diff --git a/stackoverflow-survey.ipynb b/stackoverflow-survey.ipynb index 59f75fc..103029e 100644 --- a/stackoverflow-survey.ipynb +++ b/stackoverflow-survey.ipynb @@ -91,44 +91,46 @@ "outputs": [], "source": [ "\n", - "from sklearn.linear_model import LinearRegression, LogisticRegression\n", + "from sklearn.linear_model import LinearRegression\n", "from sklearn.model_selection import train_test_split\n", - "from sklearn.metrics import root_mean_squared_error\n", - "from sklearn.model_selection import StratifiedShuffleSplit\n", + "from sklearn.metrics import root_mean_squared_error, r2_score\n", "import traceback\n", "import numpy as np\n", "\n", "# still haven't come up with a name\n", "class Foo:\n", - " def __init__(self, dataset, language, jobs=None, n_rich_outliers=0, n_poor_outliers=0, country=\"United States of America\"):\n", + " def __init__(self, dataset, language, jobs=None, \n", + " n_rich_outliers=0, n_poor_outliers=0, \n", + " country=\"United States of America\"):\n", " self.devs = None\n", " self.canvas = None\n", " self.language = language\n", " self.country = country\n", " # focus on people who have given ...\n", - " key = \"ConvertedCompYearly\"\n", - " key2 = \"YearsCodePro\"\n", - " df = dataset.dropna(subset=[key, key2])\n", - " self.key = key\n", - " self.key2 = key2\n", - "\n", - " criteria = {\"MainBranch\":\"I am a developer by profession\"}\n", - "\n", - " #print(df[\"Country\"].unique)\n", + " key_x = \"YearsCodePro\"\n", + " key_y = \"ConvertedCompYearly\"\n", + " df = dataset.dropna(subset=[key_x, key_y])\n", + " self.key_x = key_x\n", + " self.key_y = key_y\n", + " \n", + " qualifiers = {\n", + " \"MainBranch\":\"I am a developer by profession\",\n", + " }\n", " if country:\n", - " criteria[\"Country\"] = country\n", - " for k in criteria:\n", - " df = df[df[k] == criteria[k] ] \n", + " qualifiers[\"Country\"] = country\n", + " for k in qualifiers:\n", + " df = df[df[k] == qualifiers[k] ] \n", "\n", " # chatgpt tells me about filtering with multiple strings\n", " if jobs:\n", " df = df[df.isin(jobs).any(axis=1)]\n", "\n", " devs = None\n", - " if len(language) > 1:\n", - " devs = get_lang_devs(df, language)\n", - " else:\n", + " if len(language) == 1 or language in [\"Python\", \"Java\"]:\n", " devs = get_c_devs(df, lang=language)\n", + " else:\n", + " devs = get_lang_devs(df, language)\n", + " \n", " replacement_dict = {\n", " 'Less than 1 year': '0.5',\n", " 'More than 50 years': '51',\n", @@ -136,77 +138,80 @@ "\n", " # https://stackoverflow.com/questions/47443134/update-column-in-pandas-dataframe-without-warning\n", " pd.options.mode.chained_assignment = None # default='warn'\n", - " new_column = devs[key2].replace(replacement_dict)\n", - " devs[key2] = pd.to_numeric(new_column, errors='coerce')\n", + " new_column = devs[key_x].replace(replacement_dict)\n", + " devs[key_x] = pd.to_numeric(new_column, errors='coerce')\n", " pd.options.mode.chained_assignment = 'warn' # default='warn'\n", - " # print( devs[key2].unique() )\n", + " # print( devs[key_x].unique() )\n", " \n", - " indices = devs[key].nlargest(n_rich_outliers).index\n", + " indices = devs[key_y].nlargest(n_rich_outliers).index\n", " devs = devs.drop(indices)\n", - " indices = devs[key].nsmallest(n_poor_outliers).index\n", + " indices = devs[key_y].nsmallest(n_poor_outliers).index\n", " self.devs = devs.drop(indices)\n", - " del devs, new_column, criteria\n", + " del devs, new_column\n", " \n", - " def visualize(self, n_lowest=0, hue=\"Country\"): \n", + " def visualize(self, n_lowest=0, \n", + " hue=\"Country\", palette=sb.color_palette() ): \n", " self.canvas = plt.figure()\n", - " key = self.key\n", - " key2 = self.key2\n", + " key_x = self.key_x\n", + " key_y = self.key_y\n", "\n", " if n_lowest > 0:\n", " # chatgpt draws my line\n", " # Calculate the lowest nth point (for example, the 5th lowest value)\n", " # iloc[-1] gets the last element from the n smallest\n", - " lowest_nth = self.devs[key].nsmallest(n_lowest).iloc[-1] \n", + " lowest_nth = self.devs[key_y].nsmallest(n_lowest).iloc[-1] \n", " # Draw a horizontal line at the lowest nth point\n", " # label=f'Lowest {n_poorest}th Point: {lowest_nth_value:.2f}'\n", - " plt.axhline(y=lowest_nth, color='purple', linestyle='--', label=\"y=%0.2f\" % lowest_nth )\n", + " plt.axhline(y=lowest_nth, color='purple', linestyle='--', \n", + " label=\"y=%0.2f\" % lowest_nth, zorder=-1 )\n", "\n", - " sb.scatterplot(data=self.devs, x=key2, y=key, hue=hue)\n", + " sb.scatterplot(data=self.devs, x=key_x, y=key_y, hue=hue, palette=palette)\n", " plt.legend(loc='lower center', bbox_to_anchor=(1.5,0)) \n", " title = \"Annual Salary of %s Developers Over Years of Experience\" % self.language\\\n", " + \"\\nsample size=%i\" % len (self.devs)\\\n", " + \"\\ncountry=%s\" % self.country\n", " plt.title(title)\n", "\n", - " def run_regression(self, split=train_test_split, \n", - " x_transform=None, change_base=None, x_shift=0,\n", - " line_color='red'):\n", + " def run_regression(self, model=LinearRegression(), split=train_test_split, \n", + " x_transform=None, change_base=None, x_shift=0, y_shift=0,\n", + " line_color='red', random=333):\n", " df = self.devs # .sort_values(by = self.key2)\n", - "# df['binned'] = pd.qcut(df[self.key], q=4, labels=False)\n", - " X = df[self.key2].to_frame() + x_shift\n", + " X = df[self.key_x].to_frame()\n", " if x_transform is not None and change_base is not None:\n", " X = x_transform (X, a=change_base ) \n", " elif x_transform is not None:\n", " X = x_transform (X) \n", - "\n", - " y = df[self.key].to_frame()\n", - "# y = df['binned']\n", + " X = X + x_shift\n", + " y = df[self.key_y].to_frame() + y_shift\n", " \n", - " X_train, X_test, y_train, y_test = split(X, y, test_size=0.2, random_state=999)\n", + " X_train, X_test, y_train, y_test = split(X, y, test_size=0.2, random_state=random)\n", "\n", - " model = LinearRegression()\n", " model.fit(X_train, y_train)\n", " y_pred = model.predict(X_test)\n", "\n", " print(\"+----------------------+\")\n", + " print(\"%s regression line for %s\" % (line_color, self.language))\n", " print(\"coefficient =\", model.coef_)\n", " print('intercept=', model.intercept_)\n", " rmse = root_mean_squared_error(y_test, y_pred)\n", " print(\"rmse = \", rmse)\n", + " r2 = r2_score(y_test, y_pred)\n", + " print(\"r2 score = \", r2)\n", " print(\"sample predictions:\")\n", " print(y_pred[3:6])\n", " print(\"+----------------------+\")\n", " \n", " plt.figure(self.canvas)\n", - " plt.xlim(left=0, right=40) # Adjust these values as needed\n", + "\n", " plt.plot(X_test, y_pred, color=line_color, label='Regression Line')\n", " plt.legend(loc='lower center', bbox_to_anchor=(1.5,0)) \n", " del y_pred, model\n", "\n", "\n", - " def export_image(self, filename = \"images/programmers-%s-%s.png\"):\n", + " def export_image(self, base_filename = \"images/programmers-%s-%s.png\"):\n", " plt.figure(self.canvas)\n", - " plt.savefig(filename % (self.language, self.country), bbox_inches='tight')\n", + " filename = base_filename % (self.language, self.country)\n", + " plt.savefig(filename.replace(' ', '-'), bbox_inches='tight')\n", "\n", "# the higher a is, the steeper the line gets\n", "def log_base_a(x, a=1.07):\n", @@ -220,7 +225,6 @@ "metadata": {}, "outputs": [], "source": [ - "\n", "\n", "# expected python jobs\n", "pyjobs = [\"Data scientist or machine learning specialist\",\n", @@ -230,15 +234,36 @@ "# \"Developer, QA or test\"\n", "]\n", "\n", - "python = Foo(so_df, \"Python\", jobs=pyjobs, n_rich_outliers=9, n_poor_outliers=2)\n", - "python.visualize(hue=\"DevType\")\n", - "# earnings vary widely after the first year\n", - "python.run_regression( x_transform=log_base_a, x_shift=1)\n", - "python.run_regression( x_transform=log_base_a, change_base=1.2, x_shift=1, line_color='magenta')\n", - "python.run_regression( x_transform=log_base_a, change_base=1.12, x_shift=1, line_color='lightgreen')\n", + "python = Foo(so_df, \"Python\", jobs=pyjobs, n_rich_outliers=12, n_poor_outliers=2)\n", + "python.visualize(hue=\"DevType\", palette=[\"#dbdb32\", \"#34bf65\", \"#ac70e0\"])\n", + "python.run_regression()\n", + "python.run_regression( x_transform=log_base_a, change_base=1.20, \n", + " x_shift=0, y_shift=-1.5e4, line_color='cyan', random=888)\n", "python.export_image()" ] }, + { + "cell_type": "markdown", + "id": "f4e3516e-ffe3-4768-ae92-e5cb0be503f8", + "metadata": {}, + "source": [ + "## Business Understanding\n", + "\n", + "* For Python programmers specialized in data e.g data scientists, engineers, or analysts, a linear model moderately fits the relationship between income and years of experience. For data engineers and scientists, there is a possible divergence within the career path at 10 years of experience. \n", + "\n", + "* The typical starting salary within the field of data science is around 100,000 to 120,000 dollars.\n", + "\n", + "* The income of a data professional can either increase by 2,000 per year (red) or 10,000 per year (cyan).\n", + "\n", + "* For both models, the r2 score ranges from poor to moderate = 0.20 - 0.37 depending on the random number. The variability not explained by the model could be the result of the fields such as advertising, finance, or bio/medical technology.\n", + "\n", + "* For any given point in the career, the model is off by 39,000 or 42,000 dollars.\n", + "\n", + "Generally, for low uncomes poorly explained by the model, the cause could be getting a new job after a year of unemployment, internships, or part-time positions. For high incomes poorly explained by the model, the cause could be professionals at large companies who had recently added a programming language to their skill set. Other causes could be company size or working hours.\n", + "\n", + "(Business understanding was done for C first. Questions are there.)" + ] + }, { "cell_type": "code", "execution_count": null, @@ -247,18 +272,57 @@ "outputs": [], "source": [ "# expected C jobs\n", - "cjobs = [\"Developer, embedded applications or devices\", \n", - " \"Developer, game or graphics\",\n", - " \"Hardware Engineer\" ,\n", + "cjobs = [\n", + " \"Developer, embedded applications or devices\", \n", + " \"Developer, game or graphics\",\n", + " \"Hardware Engineer\" ,\n", " # \"Project manager\", \n", " # \"Product manager\"\n", "]\n", - "c = Foo(so_df, \"C\", jobs=cjobs, n_rich_outliers=11)\n", - "c.visualize(n_lowest=3, hue=\"DevType\")\n", - "c.run_regression(x_transform=log_base_a, change_base=1.25)\n", + "c = Foo(so_df, \"C\", jobs=cjobs, n_rich_outliers=30, n_poor_outliers=2)\n", + "c.visualize(n_lowest=7, hue=\"DevType\", palette=[\"#57e6da\",\"#d9e352\",\"#cc622d\"] ) \n", + "c.run_regression()\n", + "c.run_regression(x_transform=log_base_a, change_base=1.3, \n", + " x_shift=2, y_shift=-5000, line_color=\"magenta\", random=555)\n", "c.export_image()" ] }, + { + "cell_type": "markdown", + "id": "89d86a1e-dc65-48e4-adcf-bb10188fd0b7", + "metadata": {}, + "source": [ + "## Business Understanding\n", + "\n", + "1. For C programmers, specifically embedded systems, graphics, and hardware engineers, a linear model fits the relationship of income and years of experience.\n", + "\n", + "2. A coefficient = 11973.469 indicates that for each year of experience, a C programmer typically earns an additional $12,000 per year.\n", + "\n", + "3. Because the graph looks like a spray of water, after 10 years of experience, the salaries for C programming professionals strongly vary.\n", + "\n", + "4. A junior C programmer, at 2 years of experience, typically earns $54,776.266 per year.\n", + "\n", + "5. An r2 score = 0.571 indicates a bit over half of the variability in data is explained by the independent variable. This however is only for incomes below 200,000 dollars. Some participants with 5 years of professional experience were reporting incomes at or around $200,000. These were considered unusual outliers. Among the game developers, they may have independently released a game.\n", + "\n", + "rmse = 21198.612 indicates the model is off by around 21,000 dollars for a given point in a career.\n", + "\n", + "### Questions that can be answered\n", + "\n", + "* Is there a linear relationship between income and years of experience.\n", + "* Is there a point in a career where raises stop ocurring?\n", + "* What is the typical salary of a entry-level or junior C programmer?\n", + "* How much more do C programmers earn for each year of experience?\n", + "* How much of the variability is explained by the model and what factors are not considered?\n" + ] + }, + { + "cell_type": "markdown", + "id": "928f421c-1f2b-4be1-9ce2-8f3593a9a823", + "metadata": {}, + "source": [ + "Below cells generate extra or unused graphs." + ] + }, { "cell_type": "code", "execution_count": null, @@ -584,14 +648,6 @@ "# 55-64 years old\n", "# 65 years or older" ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "b954a811-e401-48dc-9ba4-263a5f2cf5c5", - "metadata": {}, - "outputs": [], - "source": [] } ], "metadata": {