668 lines
26 KiB
Plaintext
668 lines
26 KiB
Plaintext
{
|
|
"cells": [
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": null,
|
|
"id": "74e4cf25-6649-4633-89ea-03ffc2e23caa",
|
|
"metadata": {},
|
|
"outputs": [],
|
|
"source": [
|
|
"from collections import Counter\n",
|
|
"\n",
|
|
"import pandas as pd\n",
|
|
"import seaborn as sb\n",
|
|
"import matplotlib.pyplot as plt\n",
|
|
"\n",
|
|
"# avoid burning my eyes @ night\n",
|
|
"plt.style.use(\"dark_background\")"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": null,
|
|
"id": "f2b80545-2481-4ee8-8d43-ffd4a612a397",
|
|
"metadata": {},
|
|
"outputs": [],
|
|
"source": [
|
|
"FILE = \"data/survey_results_public.csv\"\n",
|
|
"so_df = pd.read_csv(FILE)\n",
|
|
"\n",
|
|
"print(so_df.keys())\n",
|
|
"so_df.describe()\n",
|
|
"\n",
|
|
"# check for people who aren't paying attention\n",
|
|
"count_not_apple = (so_df[\"Check\"] != \"Apples\").sum()\n",
|
|
"print(count_not_apple)\n",
|
|
"print(so_df.shape)\n",
|
|
"assert(count_not_apple == 0)\n",
|
|
"# print(so_df[:3])\n"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": null,
|
|
"id": "0e9b0c49-eac6-45e1-83f1-92813e734ef5",
|
|
"metadata": {},
|
|
"outputs": [],
|
|
"source": [
|
|
"# draw count plot of developers based on age\n",
|
|
"\n",
|
|
"def visualize_devs(df, lang, key=\"Age\",):\n",
|
|
" plt.figure()\n",
|
|
" plt.xticks(rotation=45)\n",
|
|
" # from:\n",
|
|
" # print(df[key].unique())\n",
|
|
" order = ['Under 18 years old', '18-24 years old', \\\n",
|
|
" '25-34 years old','35-44 years old',\\\n",
|
|
" '45-54 years old', '55-64 years old', \\\n",
|
|
" '65 years or older', 'Prefer not to say']\n",
|
|
" sb.countplot(x=key, data=df, order=order)\n",
|
|
" title=\"Ages of %s Programmers\" % lang\n",
|
|
" plt.title(title)\n",
|
|
" filename= \"images/%s-of-%s-programmers.png\" % (key, lang)\n",
|
|
" plt.savefig(filename, bbox_inches=\"tight\")\n",
|
|
"\n",
|
|
"def get_lang_devs(df, lang):\n",
|
|
" col = \"LanguageHaveWorkedWith\"\n",
|
|
" # will not work for single character languages (C, R)\n",
|
|
" # will mangle Java and JavaScript, Python and MicroPython\n",
|
|
" return df[ df[col].str.contains(lang, na=False) ] \n",
|
|
"\n",
|
|
"def get_c_devs(df, lang=\"C\"):\n",
|
|
" key = \"LanguageHaveWorkedWith\"\n",
|
|
" cdevs = []\n",
|
|
" for index, dev in df.iterrows():\n",
|
|
" try:\n",
|
|
" # split string into list\n",
|
|
" langs_used = dev[key].split(';')\n",
|
|
" if lang in langs_used:\n",
|
|
" cdevs.append(dev)\n",
|
|
" except AttributeError:\n",
|
|
"# print(dev[key])\n",
|
|
" pass\n",
|
|
" return pd.DataFrame(cdevs)"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": null,
|
|
"id": "b8212c27-6c76-4c8f-ba66-bbf1b5835c99",
|
|
"metadata": {},
|
|
"outputs": [],
|
|
"source": [
|
|
"\n",
|
|
"from sklearn.linear_model import LinearRegression\n",
|
|
"from sklearn.model_selection import train_test_split\n",
|
|
"from sklearn.metrics import root_mean_squared_error, r2_score\n",
|
|
"import traceback\n",
|
|
"import numpy as np\n",
|
|
"\n",
|
|
"# still haven't come up with a name\n",
|
|
"class Foo:\n",
|
|
" def __init__(self, dataset, language, jobs=None, \n",
|
|
" n_rich_outliers=0, n_poor_outliers=0, \n",
|
|
" country=\"United States of America\"):\n",
|
|
" self.devs = None\n",
|
|
" self.canvas = None\n",
|
|
" self.language = language\n",
|
|
" self.country = country\n",
|
|
" # focus on people who have given ...\n",
|
|
" key_x = \"YearsCodePro\"\n",
|
|
" key_y = \"ConvertedCompYearly\"\n",
|
|
" df = dataset.dropna(subset=[key_x, key_y])\n",
|
|
" self.key_x = key_x\n",
|
|
" self.key_y = key_y\n",
|
|
" \n",
|
|
" qualifiers = {\n",
|
|
" \"MainBranch\":\"I am a developer by profession\",\n",
|
|
" }\n",
|
|
" if country:\n",
|
|
" qualifiers[\"Country\"] = country\n",
|
|
" for k in qualifiers:\n",
|
|
" df = df[df[k] == qualifiers[k] ] \n",
|
|
"\n",
|
|
" # chatgpt tells me about filtering with multiple strings\n",
|
|
" if jobs:\n",
|
|
" df = df[df.isin(jobs).any(axis=1)]\n",
|
|
"\n",
|
|
" devs = None\n",
|
|
" if len(language) == 1 or language in [\"Python\", \"Java\"]:\n",
|
|
" devs = get_c_devs(df, lang=language)\n",
|
|
" else:\n",
|
|
" devs = get_lang_devs(df, language)\n",
|
|
" \n",
|
|
" replacement_dict = {\n",
|
|
" 'Less than 1 year': '0.5',\n",
|
|
" 'More than 50 years': '51',\n",
|
|
" }\n",
|
|
"\n",
|
|
" # https://stackoverflow.com/questions/47443134/update-column-in-pandas-dataframe-without-warning\n",
|
|
" pd.options.mode.chained_assignment = None # default='warn'\n",
|
|
" new_column = devs[key_x].replace(replacement_dict)\n",
|
|
" devs[key_x] = pd.to_numeric(new_column, errors='coerce')\n",
|
|
" pd.options.mode.chained_assignment = 'warn' # default='warn'\n",
|
|
" # print( devs[key_x].unique() )\n",
|
|
" \n",
|
|
" indices = devs[key_y].nlargest(n_rich_outliers).index\n",
|
|
" devs = devs.drop(indices)\n",
|
|
" indices = devs[key_y].nsmallest(n_poor_outliers).index\n",
|
|
" self.devs = devs.drop(indices)\n",
|
|
" del devs, new_column\n",
|
|
" \n",
|
|
" def visualize(self, hue=\"Country\", \n",
|
|
" palette=sb.color_palette() ): \n",
|
|
" self.canvas = plt.figure()\n",
|
|
" key_x = self.key_x\n",
|
|
" key_y = self.key_y\n",
|
|
"\n",
|
|
" sb.scatterplot(data=self.devs, x=key_x, y=key_y, hue=hue, palette=palette)\n",
|
|
" plt.legend(loc='lower center', bbox_to_anchor=(1.5,0)) \n",
|
|
" title = \"Annual Compensation of %s Programmers Over Years of Experience\" % self.language\\\n",
|
|
" + \"\\nsample size=%i\" % len (self.devs)\\\n",
|
|
" + \"\\ncountry=%s\" % self.country\n",
|
|
" plt.title(title)\n",
|
|
"\n",
|
|
" def run_regression(self, model=LinearRegression(), split=train_test_split, \n",
|
|
" x_transform=None, change_base=None, x_shift=0, y_shift=0,\n",
|
|
" line_color='red', random=333):\n",
|
|
" df = self.devs # .sort_values(by = self.key2)\n",
|
|
" X = df[self.key_x].to_frame()\n",
|
|
" if x_transform is not None and change_base is not None:\n",
|
|
" X = x_transform (X, a=change_base ) \n",
|
|
" elif x_transform is not None:\n",
|
|
" X = x_transform (X) \n",
|
|
" X = X + x_shift\n",
|
|
" y = df[self.key_y].to_frame() + y_shift\n",
|
|
" \n",
|
|
" X_train, X_test, y_train, y_test = split(X, y, test_size=0.2, random_state=random)\n",
|
|
"\n",
|
|
" model.fit(X_train, y_train)\n",
|
|
" y_pred = model.predict(X_test)\n",
|
|
" \n",
|
|
" m = model.coef_[0][0]\n",
|
|
" b = model.intercept_[0]\n",
|
|
" print(\"+----------------------+\")\n",
|
|
" print(\"%s regression line for %s\" % (line_color, self.language))\n",
|
|
" print(\"coefficient = %0.2f\" % m)\n",
|
|
" print('intercept = %0.2f' % b)\n",
|
|
" rmse = root_mean_squared_error(y_test, y_pred)\n",
|
|
" print(\"rmse = %0.2f\" % rmse)\n",
|
|
" r2 = r2_score(y_test, y_pred)\n",
|
|
" print(\"r2 score = %0.2f\" % r2)\n",
|
|
" print(\"sample predictions:\")\n",
|
|
" print(y_pred[3:6])\n",
|
|
" print(\"+----------------------+\")\n",
|
|
"\n",
|
|
" plt.figure(self.canvas)\n",
|
|
" plt.plot(X_test, y_pred, color=line_color, label='Regression Line')\n",
|
|
" plt.axhline(y=b, color=\"purple\", linestyle='--', \n",
|
|
" label=\"b=%0.2f\" % b, zorder=-1 )\n",
|
|
" plt.legend(loc='lower center', bbox_to_anchor=(1.5,0)) \n",
|
|
" del y_pred, model\n",
|
|
"\n",
|
|
"\n",
|
|
" def export_image(self, base_filename = \"images/programmers-%s-%s.png\"):\n",
|
|
" plt.figure(self.canvas)\n",
|
|
" filename = base_filename % (self.language, self.country)\n",
|
|
" plt.savefig(filename.replace(' ', '-'), bbox_inches='tight')\n",
|
|
"\n",
|
|
"# the higher a is, the steeper the line gets\n",
|
|
"def log_base_a(x, a=1.07):\n",
|
|
" return np.log10(x)/np.log(a)"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": null,
|
|
"id": "ba81c59c-0610-4f71-96fb-9eddd7736329",
|
|
"metadata": {},
|
|
"outputs": [],
|
|
"source": [
|
|
"\n",
|
|
"# expected python jobs\n",
|
|
"pyjobs = [\"Data scientist or machine learning specialist\",\n",
|
|
" \"Data or business analyst\",\n",
|
|
" \"Data engineer\",\n",
|
|
"# \"DevOps specialist\",\n",
|
|
"# \"Developer, QA or test\"\n",
|
|
"]\n",
|
|
"\n",
|
|
"python = Foo(so_df, \"Python\", jobs=pyjobs, n_rich_outliers=12, n_poor_outliers=2)\n",
|
|
"python.visualize(hue=\"DevType\", palette=[\"#dbdb32\", \"#34bf65\", \"#ac70e0\"])\n",
|
|
"python.run_regression()\n",
|
|
"python.run_regression( x_transform=log_base_a, change_base=1.20, \n",
|
|
" x_shift=0, y_shift=-1.5e4, line_color='cyan', random=888)\n",
|
|
"python.export_image()"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "markdown",
|
|
"id": "f4e3516e-ffe3-4768-ae92-e5cb0be503f8",
|
|
"metadata": {},
|
|
"source": [
|
|
"## Business Understanding\n",
|
|
"\n",
|
|
"* For Python programmers specialized in data e.g data scientists, engineers, or analysts, a linear model moderately fits the relationship between income and years of experience. For data engineers and scientists, there is a possible divergence within the career path at 10 years of experience. \n",
|
|
"\n",
|
|
"* The typical starting salary within the field of data science is around 100,000 to 120,000 dollars.\n",
|
|
"\n",
|
|
"* The income of a data professional can either increase by 2,000 per year (red) or 10,000 per year (cyan).\n",
|
|
"\n",
|
|
"* For both models, the r2 score ranges from poor to moderate = 0.20 - 0.37 depending on the random number. The variability not explained by the model could be the result of different fields that employ dats scientists/analysts/engineers such as finance, bio/med, or advertising.\n",
|
|
"\n",
|
|
"* For any given point in the career, the model is off by 39,000 or 42,000 dollars.\n",
|
|
"\n",
|
|
"Generally, for low uncomes poorly explained by the model, the cause could be getting a new job after a year of unemployment, internships, or part-time positions. For high incomes poorly explained by the model, the cause could be professionals at large companies who had recently added a programming language to their skill set. Other causes could be company size or working hours.\n",
|
|
"\n",
|
|
"(Business understanding was done for C first. Questions are there.)"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": null,
|
|
"id": "0e27f76c-8f87-4c39-ac2f-5a9b2434466f",
|
|
"metadata": {},
|
|
"outputs": [],
|
|
"source": [
|
|
"# expected C jobs\n",
|
|
"cjobs = [\n",
|
|
" \"Developer, embedded applications or devices\", \n",
|
|
" \"Developer, game or graphics\",\n",
|
|
" \"Hardware Engineer\" ,\n",
|
|
" # \"Project manager\", \n",
|
|
" # \"Product manager\"\n",
|
|
"]\n",
|
|
"c = Foo(so_df, \"C\", jobs=cjobs, n_rich_outliers=30, n_poor_outliers=2)\n",
|
|
"c.visualize(hue=\"DevType\", palette=[\"#57e6da\",\"#d9e352\",\"#cc622d\"] ) \n",
|
|
"c.run_regression()\n",
|
|
"c.run_regression(x_transform=log_base_a, change_base=1.3, \n",
|
|
" x_shift=2, y_shift=-5000, line_color=\"magenta\", random=555)\n",
|
|
"c.export_image()"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "markdown",
|
|
"id": "89d86a1e-dc65-48e4-adcf-bb10188fd0b7",
|
|
"metadata": {},
|
|
"source": [
|
|
"## Business Understanding\n",
|
|
"\n",
|
|
"1. For C programmers, specifically embedded systems, graphics, and hardware engineers, a linear model fits the relationship of income and years of experience.\n",
|
|
"\n",
|
|
"2. A coefficient = 11973.469 indicates that for each year of experience, a C programmer typically earns an additional $12,000 per year.\n",
|
|
"\n",
|
|
"3. Because the graph looks like a spray of water, after 10 years of experience, the salaries for C programming professionals strongly vary.\n",
|
|
"\n",
|
|
"4. A junior C programmer, at 2 years of experience, typically earns $54,776.266 per year.\n",
|
|
"\n",
|
|
"5. An r2 score = 0.571 indicates a bit over half of the variability in data is explained by the independent variable. This however is only for incomes below 200,000 dollars. Some participants with 5 years of professional experience were reporting incomes at or around $200,000. These were considered unusual outliers. Among the game developers, they may have independently released a game.\n",
|
|
"\n",
|
|
"rmse = 21198.612 indicates the model is off by around 21,000 dollars for a given point in a career.\n",
|
|
"\n",
|
|
"### Questions that can be answered\n",
|
|
"\n",
|
|
"* Is there a linear relationship between income and years of experience.\n",
|
|
"* Is there a point in a career where raises stop ocurring?\n",
|
|
"* What is the typical salary of a entry-level or junior C programmer?\n",
|
|
"* How much more do C programmers earn for each year of experience?\n",
|
|
"* How much of the variability is explained by the model and what factors are not considered?\n"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "markdown",
|
|
"id": "928f421c-1f2b-4be1-9ce2-8f3593a9a823",
|
|
"metadata": {},
|
|
"source": [
|
|
"Below cells generate extra or unused graphs."
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": null,
|
|
"id": "8357f841-23a0-4bfa-bf09-860bd3e014b8",
|
|
"metadata": {},
|
|
"outputs": [],
|
|
"source": [
|
|
"\n",
|
|
"jsjobs = [\"Developer, full-stack\",\n",
|
|
" \"Developer, front-end\",\n",
|
|
" \"Developer, mobile\"\n",
|
|
"]\n",
|
|
"\n",
|
|
"js = Foo(so_df, \"JavaScript\", jobs=jsjobs, n_rich_outliers=6, country=\"Ukraine\")\n",
|
|
"js.visualize(hue=\"DevType\")\n",
|
|
"js.export_image()"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": null,
|
|
"id": "11a1b9fb-db48-4749-8d77-4241a99d7bad",
|
|
"metadata": {},
|
|
"outputs": [],
|
|
"source": [
|
|
"visualize_devs( get_c_devs(so_df) , \"C\")\n",
|
|
"\n",
|
|
"for lang in [\"Cobol\", \"Prolog\", \"Ada\", \"Python\"]:\n",
|
|
" foo = get_lang_devs(so_df, lang)\n",
|
|
" visualize_devs(foo, lang)"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": null,
|
|
"id": "35b9727a-176c-4193-a1f9-a508aecd2d1c",
|
|
"metadata": {
|
|
"jupyter": {
|
|
"source_hidden": true
|
|
}
|
|
},
|
|
"outputs": [],
|
|
"source": [
|
|
"# get popularity of different programming languages\n",
|
|
"\n",
|
|
"#keys re: languages are:\n",
|
|
"#LanguageHaveWorkedWith,LanguageWantToWorkWith,LanguageAdmired,LanguageDesired\n",
|
|
"\n",
|
|
"# draw as strip chart\n",
|
|
"# https://seaborn.pydata.org/generated/seaborn.stripplot.html#seaborn.stripplot\n",
|
|
"\n",
|
|
"def get_langs(dataset, key=\"LanguageHaveWorkedWith\"):\n",
|
|
" lang_count = Counter()\n",
|
|
" assert(key in dataset.keys())\n",
|
|
" for response in dataset[key]:\n",
|
|
" if type(response) == str:\n",
|
|
" lang_count.update(response.split(';'))\n",
|
|
" langs_by_popularity = dict(\n",
|
|
" sorted(lang_count.items(), key=lambda item: item[1], reverse=True)\n",
|
|
" )\n",
|
|
" return langs_by_popularity\n",
|
|
"\n",
|
|
"def visualize_langs(langs, langs2, label1 = \"condition1\", label2 = \"condition2\", saveto=None):\n",
|
|
" DOT_COLOR1 = \"lightblue\"\n",
|
|
" DOT_COLOR2 = \"red\"\n",
|
|
" BG_COLOR = \"black\" \n",
|
|
" df = pd.DataFrame(langs.items(), columns=['Languages', 'Count'])\n",
|
|
" df2 = pd.DataFrame(langs2.items(), columns=['Languages', 'Count'])\n",
|
|
" \n",
|
|
" plt.figure(figsize=(10,15)) \n",
|
|
" \n",
|
|
" sb.stripplot(x='Count', y='Languages', data=df, \\\n",
|
|
" size=5, color=DOT_COLOR1, label=\"have worked with\", jitter=True)\n",
|
|
" sb.stripplot(x='Count', y='Languages', data=df2, \\\n",
|
|
" size=5, color=DOT_COLOR2, label=\"want to work with\", jitter=True)\n",
|
|
" \n",
|
|
" # chatgpt draws my legend\n",
|
|
" # Create custom legend handles to avoid duplicates\n",
|
|
" # color = 'w' means do not draw line bissecting point\n",
|
|
" blue_patch = plt.Line2D(\n",
|
|
" [0], [0], marker='o', color=BG_COLOR, \\\n",
|
|
" label=label1, markerfacecolor=DOT_COLOR1, markersize=10)\n",
|
|
" red_patch = plt.Line2D(\n",
|
|
" [0], [0], marker='o', color=BG_COLOR, \\\n",
|
|
" label=label2, markerfacecolor=DOT_COLOR2, markersize=10)\n",
|
|
" \n",
|
|
" # Show the legend with custom handles\n",
|
|
" plt.legend(handles=[blue_patch, red_patch], loc=\"center right\")\n",
|
|
" \n",
|
|
" plt.grid(axis='x', linestyle='--', alpha=0.75) \n",
|
|
" plt.title(\"%s vs %s\" % (label1, label2))\n",
|
|
" if saveto is not None:\n",
|
|
" plt.savefig(saveto, bbox_inches='tight')\n",
|
|
" del df, df2\n",
|
|
"\n",
|
|
"l1 = get_langs( so_df )\n",
|
|
"l2 = get_langs( so_df, \"LanguageWantToWorkWith\" )\n",
|
|
"visualize_langs(l1,l2, \n",
|
|
" label1=\"have worked with\", label2=\"want to work with\",\n",
|
|
" saveto=\"images/used-vs-want2use.png\")\n",
|
|
"\n",
|
|
"l3 = get_langs( so_df, \"LanguageAdmired\")\n",
|
|
"l4 = get_langs( so_df, \"LanguageWantToWorkWith\")\n",
|
|
"visualize_langs(l3, l4, \n",
|
|
" label1=\"admired\", label2=\"want to work with\",\n",
|
|
" saveto=\"images/admired-vs-want2use.png\")\n",
|
|
" "
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": null,
|
|
"id": "d0bfdb92-378a-4452-91cc-4d21afd2d6cc",
|
|
"metadata": {
|
|
"jupyter": {
|
|
"source_hidden": true
|
|
}
|
|
},
|
|
"outputs": [],
|
|
"source": [
|
|
"# draw horizontal bar plot\n",
|
|
"# https://seaborn.pydata.org/examples/part_whole_bars.html\n",
|
|
"\n",
|
|
"# investigate extrinsic vs intrinsic motivation\n",
|
|
"def get_difference(dict1, dict2, proportion=False):\n",
|
|
" keys = dict1.keys()\n",
|
|
" result = dict()\n",
|
|
" for key in keys:\n",
|
|
" if proportion:\n",
|
|
" result[key] = round((dict1[key] - dict2[key])/dict2[key],2)\n",
|
|
" else:\n",
|
|
" result[key] = dict1[key] - dict2[key]\n",
|
|
" return result\n",
|
|
"\n",
|
|
"def visualize_diff(diff_dict, color=\"lightblue\", saveto=None):\n",
|
|
" diff_sorted = dict(\n",
|
|
" sorted(diff_dict.items(), key=lambda item: item[1], reverse=True)\n",
|
|
" )\n",
|
|
" KEY = \"Value\"\n",
|
|
" df = pd.DataFrame(diff_sorted.items(), columns=['Languages', 'Value'])\n",
|
|
" plt.figure(figsize=(15,20)) \n",
|
|
" sb.barplot(x=KEY, y='Languages', data=df, color=color)\n",
|
|
" DELTA = '\\u0394'\n",
|
|
" for index, value in enumerate(df[KEY]):\n",
|
|
" # chatgpt annotates my chart\n",
|
|
" # Position the text at the base of the bar\n",
|
|
" if value >= 0:\n",
|
|
" # Adjust the x position for positive values\n",
|
|
" plt.text(value, index, DELTA+str(value), va='center', ha=\"left\") \n",
|
|
" else:\n",
|
|
" # Adjust the x position for negative values\n",
|
|
" plt.text(value, index, DELTA+str(value), va='center', ha='right') \n",
|
|
" lowest = 0\n",
|
|
" offset = 0\n",
|
|
" positive_values = df[df[KEY] > 0][KEY]\n",
|
|
" if not positive_values.empty:\n",
|
|
" lowest = positive_values.min()\n",
|
|
" offset = list(positive_values).count(lowest) \n",
|
|
" if len(positive_values) < len(df):\n",
|
|
" # don't draw the line if every value is greater than 0_\n",
|
|
" plt.axhline(y=df[KEY].tolist().index(lowest) + (offset-0.5), \n",
|
|
" color='red', linestyle='--', zorder=-1)\n",
|
|
" if saveto is not None:\n",
|
|
" plt.savefig(saveto, bbox_inches='tight')\n",
|
|
" \n",
|
|
"motiv_diff = get_difference(l2, l1, proportion=True)\n",
|
|
"# print(motiv_diff)\n",
|
|
"visualize_diff(motiv_diff, saveto=\"images/delta.png\")\n",
|
|
"motiv_diff = get_difference(l2, l1)\n",
|
|
"visualize_diff(motiv_diff, saveto=\"images/delta-b.png\")\n",
|
|
"\n",
|
|
"# no clear description of what \"admired\" is\n",
|
|
"# in the schema\n",
|
|
"# but generally people want to use the languages\n",
|
|
"# they admire\n",
|
|
"\n",
|
|
"# determine level of hype\n",
|
|
"# hype = get_difference(l4, l3)\n",
|
|
"# print(hype)\n",
|
|
"# visualize_diff(hype, color=\"red\")"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": null,
|
|
"id": "f6b1a935-eeda-416f-8adf-5e854d3aa066",
|
|
"metadata": {
|
|
"jupyter": {
|
|
"source_hidden": true
|
|
}
|
|
},
|
|
"outputs": [],
|
|
"source": [
|
|
"# do people fall out of love with langs\n",
|
|
"# the more they are used professionally?\n",
|
|
"\n",
|
|
"def visualize_favor(df, key_x, key_y, MAGIC_X=0, MAGIC_Y=0, title=str(), saveto=None):\n",
|
|
" plt.figure()\n",
|
|
" OFFSET = 1 # push text away from point slightly\n",
|
|
" for i in range(merged.shape[0]):\n",
|
|
" # label points that aren't un a cluster\n",
|
|
" if merged[key_x][i] > MAGIC_X or merged[key_y][i] > MAGIC_Y:\n",
|
|
" plt.text(merged[key_x].iloc[i]+OFFSET, \n",
|
|
" merged[key_y].iloc[i]+OFFSET, \n",
|
|
" merged[\"Language\"].iloc[i], \n",
|
|
" ha=\"left\",\n",
|
|
" size='medium')\n",
|
|
"\n",
|
|
" sb.scatterplot(data=merged, x=key_x, y=key_y, hue=\"Language\")\n",
|
|
" plt.legend(loc='lower left', bbox_to_anchor=(0, -1.25), ncol=3) \n",
|
|
" plt.title(title)\n",
|
|
" if saveto is not None:\n",
|
|
" plt.savefig(saveto, bbox_inches='tight')\n",
|
|
" pass\n",
|
|
"key_x = \"Users\"\n",
|
|
"key_y = \"Potential '\\u0394'Users\"\n",
|
|
"df1 = pd.DataFrame(l1.items(), columns=['Language', key_x])\n",
|
|
"df2 = pd.DataFrame(motiv_diff.items(), columns=['Language', key_y])\n",
|
|
"# chatgpt tells me how to combine df\n",
|
|
"merged = pd.merge(df1, df2[[\"Language\", key_y]], on='Language', how='left')\n",
|
|
"visualize_favor(merged, key_x, key_y, \n",
|
|
" MAGIC_X=5000, MAGIC_Y=2000, \n",
|
|
" saveto=\"images/favor.png\")\n",
|
|
"del df1, df2, merged"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": null,
|
|
"id": "e90cf119-c50d-468a-bc87-72dac41176ce",
|
|
"metadata": {
|
|
"jupyter": {
|
|
"source_hidden": true
|
|
},
|
|
"scrolled": true
|
|
},
|
|
"outputs": [],
|
|
"source": [
|
|
"# see how much money are people making\n",
|
|
"\n",
|
|
"def get_mean_by_category(df, category, key=\"ConvertedCompYearly\"):\n",
|
|
" unique = df[category].unique()\n",
|
|
" result = dict()\n",
|
|
" for u in unique:\n",
|
|
" mean = df[df[category] == u][key].mean()\n",
|
|
" result[u] = mean\n",
|
|
" return result\n",
|
|
"\n",
|
|
"def show_me_the_money(df, saveto=None):\n",
|
|
" key_x = \"ConvertedCompYearly\"\n",
|
|
" key_y = \"DevType\"\n",
|
|
" \n",
|
|
" means = get_mean_by_category(df, key_y) \n",
|
|
" mean_df = pd.DataFrame(means.items(), columns=[key_y, key_x])\n",
|
|
"\n",
|
|
" plt.figure(figsize=(14,18)) \n",
|
|
" plt.axvline(x=1e5, color='red', linestyle='--', label=\"x = $100,000\")\n",
|
|
" plt.axvline(x=1e6, color='lightgreen', linestyle='--', label=\"x = millionaire\")\n",
|
|
" sb.barplot(x=key_x, y=key_y, data=mean_df.sort_values(by=key_x), \\\n",
|
|
" color='lavender', alpha=0.7, label=\"average compensation\")\n",
|
|
" sb.stripplot(x=key_x, y=key_y, data=df, \\\n",
|
|
" size=3, jitter=True)\n",
|
|
" if saveto is not None:\n",
|
|
" plt.savefig(saveto, bbox_inches='tight')\n",
|
|
" \n",
|
|
"# print survey ans\n",
|
|
"#employment_status = Counter(so_df[\"MainBranch\"])\n",
|
|
"#print(employment_status)\n",
|
|
"\n",
|
|
"#employment_type = Counter(so_df[\"DevType\"])\n",
|
|
"#print(employment_type)\n",
|
|
"\n",
|
|
"key = \"ConvertedCompYearly\"\n",
|
|
"# answers = so_df[:-1][key].count()\n",
|
|
"# print(answers, \"people answered re: \", key)\n",
|
|
"df_no_na = so_df.dropna(subset=[key])\n",
|
|
"indices = df_no_na[key].nlargest(15).index\n",
|
|
"\n",
|
|
"show_me_the_money( df_no_na.drop(indices), saveto=\"images/compensation-by-profession.png\" )\n",
|
|
"# could also ask myself what portion of developers \n",
|
|
"# earn less than the mean compensation\n",
|
|
"# (what titles have high standard deviations in earnings)"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": null,
|
|
"id": "cdf21b1c-1316-422f-ad14-48150f80366c",
|
|
"metadata": {
|
|
"jupyter": {
|
|
"source_hidden": true
|
|
}
|
|
},
|
|
"outputs": [],
|
|
"source": [
|
|
"\n",
|
|
"# key = \"DevType\"\n",
|
|
"# prof = \"Developer, full-stack\"\n",
|
|
"\n",
|
|
"key = \"MainBranch\"\n",
|
|
"prof = \"I am a developer by profession\"\n",
|
|
"col = \"ConvertedCompYearly\"\n",
|
|
"\n",
|
|
"devs = df_no_na[df_no_na[key] == prof ] \n",
|
|
"pd.set_option('display.float_format', '{:.2f}'.format)\n",
|
|
"devs.describe()[col]\n",
|
|
"\n",
|
|
"# who the hell is making $1/yr \n",
|
|
"# devs[devs[col] == 1.0]\n",
|
|
"\n",
|
|
"# who are the millionaires\n",
|
|
"# devs[devs[col] > 1e6]\n",
|
|
"\n",
|
|
"# who make more than the mean\n",
|
|
"# devs[devs[col] > 76230.84]\n",
|
|
"\n",
|
|
"# who make more than the median\n",
|
|
"# devs[devs[col] > 63316.00]\n",
|
|
"\n",
|
|
"# the ancient ones\n",
|
|
"so_df[so_df[\"YearsCodePro\"] == 'More than 50 years']\n",
|
|
"# should drop the 18-24 year old who is either bullshitting or recalls a past life\n",
|
|
"# 55-64 years old\n",
|
|
"# 65 years or older"
|
|
]
|
|
}
|
|
],
|
|
"metadata": {
|
|
"kernelspec": {
|
|
"display_name": "Python 3 (ipykernel)",
|
|
"language": "python",
|
|
"name": "python3"
|
|
},
|
|
"language_info": {
|
|
"codemirror_mode": {
|
|
"name": "ipython",
|
|
"version": 3
|
|
},
|
|
"file_extension": ".py",
|
|
"mimetype": "text/x-python",
|
|
"name": "python",
|
|
"nbconvert_exporter": "python",
|
|
"pygments_lexer": "ipython3",
|
|
"version": "3.12.7"
|
|
}
|
|
},
|
|
"nbformat": 4,
|
|
"nbformat_minor": 5
|
|
}
|