scuti/stacksurvey
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Compare commits

base: scuti:311db886a4476fc0394f24345d8fee9c846d6893
Branches Tags
scuti:master scuti:write-readme
...
compare: scuti:f49283a7ccef2bd482417f7fbd3b90df09517d61
Branches Tags
scuti:master scuti:write-readme

5 Commits
311db886a4 ... f49283a7cc

Author SHA1 Message Date
scuti
f49283a7cc Added train-test splitting for log regression. 2025-04-28 06:31:35 -07:00
scuti
1f7fe33915 Added function show_model_stats(). 2025-04-27 16:23:33 -07:00
scuti
3c3e804251 Simplified code. Removed to_frame(). 2025-04-27 12:42:39 -07:00
scuti
67d1441303 Added logarithmic regression 
Not part of the course but fits better.
 2025-04-27 10:49:37 -07:00
scuti
b18a5cb42a Implemented "risky" (pink) model. 
Also cleaned up code.
Training on sorted data is unrecommended and "risky";
however, the risky model appears to be generalizing
across random state.
 2025-04-27 10:06:33 -07:00
1 changed files with 92 additions and 31 deletions
Expand all files Collapse all files

123
stackoverflow-survey.ipynb
View File

@@ -120,8 +120,9 @@
"outputs": [],
"source": [
"visualize_devs( get_c_devs(so_df) , 'C')\n",
"visualize_devs( get_c_devs(so_df, lang='Python') , 'Python')\n",
"\n",
"for lang in ['Cobol', 'Prolog', 'Ada', 'Python']:\n",
"for lang in ['Cobol', 'Prolog', 'Ada']:\n",
" foo = get_lang_devs(so_df, lang)\n",
" visualize_devs(foo, lang)"
]
@@ -215,50 +216,102 @@
" + '\\ncountry=%s' % self.country\n",
" plt.title(title)\n",
"\n",
" def run_regression(self, model=LinearRegression(), split=train_test_split, \n",
" x_transform=None, change_base=None, x_shift=0, y_shift=0,\n",
" line_color='red', random=333):\n",
" def run_regression(self, x_transform=None, change_base=1.07, \n",
" x_shift=0, y_shift=0,\n",
" random=333, risky=0,\n",
" color='red', name='Regression Line' ):\n",
" df = self.devs # .sort_values(by = self.key2)\n",
" X = df[self.key_x].to_frame()\n",
" if x_transform is not None and change_base is not None:\n",
" X = x_transform (X, a=change_base ) \n",
" elif x_transform is not None:\n",
" X = x_transform (X) \n",
" X = X + x_shift\n",
" y = df[self.key_y].to_frame() + y_shift\n",
" \n",
" X_train, X_test, y_train, y_test = split(X, y, test_size=0.2, random_state=random)\n",
" X = df[[self.key_x]]\n",
" y = df[[self.key_y]]\n",
"\n",
" # not recommended\n",
" # carries risk of model training on sorted order\n",
" # however it appears to be generalizing well\n",
" # across random state and shuffle (=True, default)\n",
" style = '-'\n",
" if risky > 0:\n",
" X = X.sort_values(by=self.key_x)\n",
" style = '--'\n",
" if risky > 1:\n",
" y = y.sort_values(by=self.key_y)\n",
" if x_transform is not None:\n",
" X = x_transform (X, a=change_base ) \n",
"\n",
" X = X + x_shift\n",
" y = y + y_shift\n",
" \n",
" X_train, X_test, y_train, y_test = train_test_split(\n",
" X, y, \n",
" test_size=0.2, \n",
" random_state=random)\n",
"\n",
" model = LinearRegression()\n",
" model.fit(X_train, y_train)\n",
" y_pred = model.predict(X_test)\n",
" \n",
" m = model.coef_[0][0]\n",
" b = model.intercept_[0]\n",
" print('+----------------------+')\n",
" print('%s regression line for %s' % (line_color, self.language))\n",
" print('coefficient = %0.2f' % m)\n",
" print('intercept = %0.2f' % b)\n",
" rmse = root_mean_squared_error(y_test, y_pred)\n",
" print('rmse = %0.2f' % rmse)\n",
" r2 = r2_score(y_test, y_pred)\n",
" print('r2 score = %0.2f' % r2)\n",
" print('sample predictions:')\n",
" print(y_pred[3:6])\n",
" print('+----------------------+')\n",
" label = '%s log regression line for %s' % (color, self.language)\n",
" show_model_stats(m, b, y_test, y_pred, label)\n",
"\n",
" plt.figure(self.canvas)\n",
" plt.plot(X_test, y_pred, color=line_color, label='Regression Line')\n",
" plt.plot(X_test, y_pred, color=color, label=name, linestyle=style)\n",
" plt.axhline(y=b, color='purple', linestyle='--', \n",
" label='b=%0.2f' % b, zorder=-1 )\n",
" plt.legend(loc='lower center', bbox_to_anchor=(1.5,0)) \n",
" del y_pred, model\n",
" del y_pred, model, X, y\n",
"\n",
" def run_log_regression(self, color='pink', nodraw=True):\n",
" df = self.devs\n",
" X = df[[self.key_x]] #.sort_values(by=self.key_x)\n",
" y = df[[self.key_y]] #.sort_values(by=self.key_y)\n",
"\n",
" X_train, X_test, y_train, y_test = train_test_split(\n",
" X, y, \n",
" test_size=0.2, \n",
" random_state=777)\n",
" \n",
" X_train_log = np.log(X_train)\n",
" X_test_log = np.log(X_test)\n",
" \n",
" # X_train_log = X_train_log.sort_values(by=self.key_x)\n",
" # y_train = y_train.sort_values(by=self.key_y)\n",
" X_test_log = X_test_log.sort_values(by=self.key_x)\n",
" X_test = X_test.sort_values(by=self.key_x)\n",
" y_test = y_test.sort_values(by=self.key_y)\n",
" \n",
" model = LinearRegression()\n",
" model.fit(X_train_log, y_train)\n",
" y_pred = model.predict(X_test_log)\n",
" y_pred.sort()\n",
"\n",
" m = model.coef_[0][0]\n",
" b = model.intercept_[0]\n",
" label = '%s log regression line for %s' % (color, self.language)\n",
" show_model_stats(m, b, y_test, y_pred, label)\n",
"\n",
" if nodraw:\n",
" return\n",
" plt.plot(X_test, y_pred, color=color, label=\"Log regression\")\n",
" plt.legend(loc='lower center', bbox_to_anchor=(1.5,0)) \n",
"\n",
" def export_image(self, base_filename = 'images/programmers-%s-%s.png'):\n",
" plt.figure(self.canvas)\n",
" filename = base_filename % (self.language, self.country)\n",
" plt.savefig(filename.replace(' ', '-'), bbox_inches='tight')\n",
"\n",
"def show_model_stats(coef, intercept, y_test, y_pred, label):\n",
" print('+----------------------+')\n",
" print(label)\n",
" print('coefficient = %0.2f' % coef)\n",
" print('intercept = %0.2f' % intercept)\n",
" rmse = root_mean_squared_error(y_test, y_pred)\n",
" print('rmse = %0.2f' % rmse)\n",
" r2 = r2_score(y_test, y_pred)\n",
" print('r2 score = %0.2f' % r2)\n",
" print('sample predictions:')\n",
" print(y_pred[3:6])\n",
" print('+----------------------+')\n",
"\n",
"# the higher a is, the steeper the line gets\n",
"def log_base_a(x, a=1.07):\n",
@@ -293,9 +346,14 @@
"\n",
"python = Foo(so_df, 'Python', jobs=pyjobs, n_rich_outliers=12, n_poor_outliers=2)\n",
"python.visualize(hue='DevType', palette=['#dbdb32', '#34bf65', '#ac70e0'])\n",
"python.run_regression()\n",
"python.run_regression(name = 'Default regression line')\n",
"python.run_regression( x_transform=log_base_a, change_base=1.20, \n",
" x_shift=0, y_shift=-1.5e4, line_color='cyan', random=888)\n",
" x_shift=0, y_shift=-1.5e4, random=888,\n",
" color='cyan', name='Tuned regression line')\n",
"\n",
"#python.run_regression(x_transform=log_base_a, change_base=1.20, risky=2, random=555, \n",
"# color='pink', name='Risky regression line')\n",
"python.run_log_regression(nodraw=False)\n",
"python.export_image()"
]
},
@@ -350,14 +408,17 @@
"c.visualize(hue='DevType', palette=['#57e6da','#d9e352','#cc622d'] ) \n",
"c.run_regression()\n",
"c.run_regression(x_transform=log_base_a, change_base=1.3, \n",
" x_shift=2, y_shift=-5000, line_color='magenta', random=555)\n",
" x_shift=2, y_shift=-5000, color='magenta', random=555)\n",
"c.run_log_regression(nodraw=False)\n",
"c.export_image()"
]
},
{
"cell_type": "markdown",
"id": "b7026c56-3049-4e60-bbc6-ee548ff58297",
"metadata": {},
"metadata": {
"jp-MarkdownHeadingCollapsed": true
},
"source": [
"## Evaluation for C\n",
"\n",
@@ -712,7 +773,7 @@
"name": "python",
"nbconvert_exporter": "python",
"pygments_lexer": "ipython3",
"version": "3.12.7"
"version": "3.13.3"
}
},
"nbformat": 4,