diff --git a/AI4Forensics/IP_Identifier_Fine_Tuning/IP_Identifer_Fine_Tuning.pptx b/AI4Forensics/IP_Identifier_Fine_Tuning/IP_Identifer_Fine_Tuning.pptx index 0de561d..af2bdb4 100644 Binary files a/AI4Forensics/IP_Identifier_Fine_Tuning/IP_Identifer_Fine_Tuning.pptx and b/AI4Forensics/IP_Identifier_Fine_Tuning/IP_Identifer_Fine_Tuning.pptx differ diff --git a/AI4Forensics/IP_Identifier_Fine_Tuning/code/compute_classification_metrics.ipynb b/AI4Forensics/IP_Identifier_Fine_Tuning/code/compute_classification_metrics.ipynb new file mode 100644 index 0000000..366231f --- /dev/null +++ b/AI4Forensics/IP_Identifier_Fine_Tuning/code/compute_classification_metrics.ipynb @@ -0,0 +1,179 @@ +{ + "cells": [ + { + "cell_type": "code", + "execution_count": 11, + "metadata": {}, + "outputs": [], + "source": [ + "import openai\n", + "import pandas as pd\n", + "import os\n", + "\n", + "openai.api_key = \"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\"\n", + "\n", + "# Run the command in command prompt (Windows or Linux)\n", + "# Make sure in same folder with the dataset_train.jsonl and dataset_test.josnl\n", + "# !openai --api-key sk-Uf4wFlyDcZwabpuZyvhxT3BlbkFJDwBxxxx api fine_tunes.create -t \"./dataset_train.jsonl\" -v \"./dataset_test.jsonl\" --compute_classification_metrics --classification_positive_class \" positive\" -m ada \n", + "\n", + "# Run the command in command prompt (Windows or Linux)\n", + "# Download the fine-tuned metrics of each training step to a text file for further analysis \n", + "# !openai -k sk-Uf4wFlyDcZwabpuZyvhxT3BlbkFJDwBciwLz1xxxxx api fine_tunes.results -i ft-UjMMhr7J7K5pBVJMH7lTO5Oe > result.csv" + ] + }, + { + "cell_type": "code", + "execution_count": 9, + "metadata": {}, + "outputs": [ + { + "data": { + "text/html": [ + "
\n", + "\n", + "\n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + "
stepelapsed_tokenselapsed_examplestraining_losstraining_sequence_accuracytraining_token_accuracyvalidation_lossvalidation_sequence_accuracyvalidation_token_accuracyclassification/accuracyclassification/precisionclassification/recallclassification/aurocclassification/auprcclassification/f1.0
732733346617330.0037271.01.0NaNNaNNaN0.9782610.9714291.00.9975490.9991470.985507
\n", + "
" + ], + "text/plain": [ + " step elapsed_tokens elapsed_examples training_loss \\\n", + "732 733 34661 733 0.003727 \n", + "\n", + " training_sequence_accuracy training_token_accuracy validation_loss \\\n", + "732 1.0 1.0 NaN \n", + "\n", + " validation_sequence_accuracy validation_token_accuracy \\\n", + "732 NaN NaN \n", + "\n", + " classification/accuracy classification/precision classification/recall \\\n", + "732 0.978261 0.971429 1.0 \n", + "\n", + " classification/auroc classification/auprc classification/f1.0 \n", + "732 0.997549 0.999147 0.985507 " + ] + }, + "execution_count": 9, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "import pandas as pd\n", + "\n", + "results = pd.read_csv('result.csv')\n", + "results[results['classification/accuracy'].notnull()].tail(1)" + ] + }, + { + "cell_type": "code", + "execution_count": 10, + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "" + ] + }, + "execution_count": 10, + "metadata": {}, + "output_type": "execute_result" + }, + { + "data": { + "image/png": "iVBORw0KGgoAAAANSUhEUgAAAiwAAAGfCAYAAAB8wYmvAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjUuMSwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/YYfK9AAAACXBIWXMAAA9hAAAPYQGoP6dpAAA3NUlEQVR4nO3deXxU9b3/8Xf2CYQkQDYSQkIQRQQCEohhsb01bRR7pVxrqRcFg8DFQquX3lpoKXD9PWy8j97ykFqvyiJSsBUVpdYqFmNFguyLguwmJCFkYcsK2Wa+vz8CY1MWMyHJmZm8no/HeTx05jvD5xwH5u3hnHd8jDFGAAAAbszX6gEAAAC+DoEFAAC4PQILAABwewQWAADg9ggsAADA7RFYAACA2yOwAAAAt0dgAQAAbo/AAgAA3B6BBQAAuD3/1rzo+eef129+8xuVlJQoOTlZzz33nEaOHHnVtQ0NDcrKytKqVatUVFSkW265Rf/zP/+ju+++27nGbrdr0aJFWrNmjUpKShQbG6tHHnlE8+fPl4+PT4tmcjgcOnXqlLp169bi1wAAAGsZY1RVVaXY2Fj5+l7nPIpx0WuvvWYCAwPNyy+/bL744gszffp0Ex4ebkpLS6+6/sknnzSxsbHmr3/9q/nyyy/N//3f/xmbzWb27NnjXPP000+bnj17mnfffdfk5eWZN954w4SEhJglS5a0eK7CwkIjiY2NjY2Njc0Dt8LCwut+z/sY49oPP0xNTdWIESP0+9//XlLTmY34+Hj9+Mc/1ty5c69YHxsbq1/+8peaNWuW87H7779fwcHBWrNmjSTpu9/9rqKjo7VixYprrvk6FRUVCg8PV2FhoUJDQ13ZJQAAYJHKykrFx8ervLxcYWFh11zn0l8J1dfXa/fu3Zo3b57zMV9fX6Wnp2vr1q1XfU1dXZ1sNluzx4KDg5WTk+P891GjRmnp0qU6evSobr75Zn322WfKycnR4sWLrzlLXV2d6urqnP9eVVUlSQoNDSWwAADgYb7ucg6XAsuZM2dkt9sVHR3d7PHo6GgdPnz4qq/JyMjQ4sWLdeedd6pfv37Kzs7WW2+9Jbvd7lwzd+5cVVZWasCAAfLz85PdbtfTTz+tSZMmXXOWrKws/fd//7cr4wMAAA/V7ncJLVmyRP3799eAAQMUGBio2bNnKzMzs9mFNa+//rpeffVV/fGPf9SePXu0atUq/e///q9WrVp1zfedN2+eKioqnFthYWF77woAALCIS2dYIiIi5Ofnp9LS0maPl5aWKiYm5qqviYyM1Pr161VbW6uzZ88qNjZWc+fOVVJSknPNz372M82dO1c//OEPJUmDBw9Wfn6+srKyNGXKlKu+b1BQkIKCglwZHwAAeCiXzrAEBgZq+PDhys7Odj7mcDiUnZ2ttLS0677WZrMpLi5OjY2NWrduncaPH+987sKFC1fcyuTn5yeHw+HKeAAAwEu53MMyZ84cTZkyRSkpKRo5cqSeffZZ1dTUKDMzU5I0efJkxcXFKSsrS5K0fft2FRUVaejQoSoqKtKiRYvkcDj05JNPOt/zX//1X/X000+rT58+uu2227R3714tXrxYU6dObaPdBAAAnszlwDJx4kSdPn1aCxYsUElJiYYOHaoNGzY4L8QtKChodraktrZW8+fPV25urkJCQjRu3DitXr1a4eHhzjXPPfecfvWrX+lHP/qRysrKFBsbq//4j//QggULbnwPAQCAx3O5h8VdVVZWKiwsTBUVFdzWDACAh2jp9zc/SwgAALg9AgsAAHB7BBYAAOD2CCwAAMDtEVgAAIDbc/m2ZgBt53hZtdbuLFCjwytu1gPg5eZ8+2Z1swVY8msTWACLGGP009f36bOTFVaPAgAt8tg3+xFYgM5mR945fXayQkH+vnp0TF99zU9WBwDLdQm0LjYQWACLLNucK0m6f3hvPXn3AIunAQD3xkW3gAWOl1Xrw0Nl8vGRHh3T1+pxAMDtEVgAC6zIaTq7kn5rtPpFhlg8DQC4PwIL0MFOV9Vp3Z4iSdKMO5MsngYAPAOBBehgq7eeUH2jQ0Pjw5WS0N3qcQDAIxBYgA50sd6uP2zLl9R0dsWHW4MAoEUILEAHenN3ocovNCi+R7AybouxehwA8BgEFqCD2B1Gy3PyJEnTxiTJz5ezKwDQUgQWoINsPFii/LMXFBYcoAdSels9DgB4FAIL0EGWftJ0K/PDdyRY2hYJAJ6IwAJ0gN3557SnoFyBfr6aPCrB6nEAwOMQWIAOcPnsyoRhcYrqZrN4GgDwPAQWoJ3lnanR3w6WSpKmjaWGHwBag8ACtLMVObkyRvrWgCj1j+5m9TgA4JEILEA7Oltdpzd2nZQkTR9LDT8AtBaBBWhHa7YVqK7RocFxYbojqYfV4wCAxyKwAO2ktsGuP2w9IUmaTg0/ANwQAgvQTt7aU6SzNfWKCw/WuEHU8APAjSCwAO3A4TBavrnpVuapY/rK34/fagBwI/hTFGgH2YfLlHumRt1s/po4It7qcQDA4xFYgHaw7FJR3KTUBIUEUcMPADeKwAK0sb0F57XjxDkF+PnokVGJVo8DAF6BwAK0seWb8yRJ9yXHKSaMGn4AaAsEFqANFZy9oPcPFEuSpt9JDT8AtBUCC9CGXt6SJ4eR7rw5UgNiQq0eBwC8BoEFaCPlF+r1+q5CSdIMavgBoE0RWIA28ur2Al2ot+vWXqEafVNPq8cBAK9CYAHaQF2jXa98ekKSNOPOvtTwA0AbI7AAbeDP+07pdFWdYkJt+u6QWKvHAQCvQ2ABbpAxxlkUN3VMogKo4QeANsefrMAN+vjoaR0rq1ZIkL9+OLKP1eMAgFcisAA36PLZlQdHxivUFmDxNADgnQgswA04UFShT788K39fH2WOpigOANoLgQW4Acs2N51d+e6QXooND7Z4GgDwXgQWoJWKyi/q3c+bavinURQHAO2KwAK00sqcPNkdRqNv6qlBcWFWjwMAXo3AArRCxcUG/WlHgSRpOmdXAKDdEViAVnhtR4Fq6u26JbqbvnFzpNXjAIDXI7AALqpvdGjllhOSpGljqeEHgI5AYAFc9O7np1RSWauobkG6byg1/ADQEQgsgAuMMVp6qShuyqhEBfn7WTwRAHQOBBbABTnHz+hwSZW6BPppUio1/ADQUQgsgAsun135QUq8wrsEWjwNAHQeBBaghQ4VV2rzsTPy9ZEeHUMNPwB0JAIL0EKXa/jvGdxL8T26WDwNAHQuBBagBYorLuqdfackSTMoigOADkdgAVrglU9PqNFhNLJvDyXHh1s9DgB0OgQW4GtU1Tboj9uaavg5uwIA1iCwAF9j7c5CVdU1Kimyq741IMrqcQCgUyKwANfRYP+qhn/62CT5+lLDDwBWILAA1/He/mIVlV9UREigJgyLs3ocAOi0CCzANRhjnLcyT05LlC2AGn4AsAqBBbiGrblndaCoUrYAXz10R4LV4wBAp0ZgAa5h2aUa/geGx6tHV2r4AcBKBBbgKo6WVunvR07Lhxp+AHALBBbgKpZfunYlY2CMEiO6WjwNAIDAAvyTsspard/bVMM//U6K4gDAHRBYgH+yausJ1dsdGp7QXcMTuls9DgBArQwszz//vBITE2Wz2ZSamqodO3Zcc21DQ4Oeeuop9evXTzabTcnJydqwYUOzNYmJifLx8blimzVrVmvGA1qtpq5Ray7V8E+nhh8A3IbLgWXt2rWaM2eOFi5cqD179ig5OVkZGRkqKyu76vr58+frpZde0nPPPaeDBw9q5syZmjBhgvbu3etcs3PnThUXFzu3jRs3SpIeeOCBVu4W0Dpv7CpUxcUGJfbsom8PjLZ6HADAJT7GGOPKC1JTUzVixAj9/ve/lyQ5HA7Fx8frxz/+sebOnXvF+tjYWP3yl79sdrbk/vvvV3BwsNasWXPVX+OJJ57Qu+++q2PHjsnHp2VV6JWVlQoLC1NFRYVCQ0Nd2SVAktRod+hffvuxCs9d1P/73iA9TPcKALS7ln5/u3SGpb6+Xrt371Z6evpXb+Drq/T0dG3duvWqr6mrq5PNZmv2WHBwsHJycq75a6xZs0ZTp069blipq6tTZWVlsw24ER98UarCcxfVvUuAvn97b6vHAQD8A5cCy5kzZ2S32xUd3fxUeXR0tEpKSq76moyMDC1evFjHjh2Tw+HQxo0b9dZbb6m4uPiq69evX6/y8nI98sgj150lKytLYWFhzi0+Pt6VXQGaMcZo6SdfSpIeTktUcCA1/ADgTtr9LqElS5aof//+GjBggAIDAzV79mxlZmbK1/fqv/SKFSt0zz33KDY29rrvO2/ePFVUVDi3wsLC9hgfncTOE+f12ckKBfr7anIafxUEAO7GpcASEREhPz8/lZaWNnu8tLRUMTExV31NZGSk1q9fr5qaGuXn5+vw4cMKCQlRUtKVd2Dk5+frww8/1LRp0752lqCgIIWGhjbbgNZaeqmG//7beysiJMjiaQAA/8ylwBIYGKjhw4crOzvb+ZjD4VB2drbS0tKu+1qbzaa4uDg1NjZq3bp1Gj9+/BVrVq5cqaioKN17772ujAXckC9PV+vDQ00hfNpYavgBwB35u/qCOXPmaMqUKUpJSdHIkSP17LPPqqamRpmZmZKkyZMnKy4uTllZWZKk7du3q6ioSEOHDlVRUZEWLVokh8OhJ598stn7OhwOrVy5UlOmTJG/v8tjAa22fHOeJCn91mj1iwyxeBoAwNW4nAwmTpyo06dPa8GCBSopKdHQoUO1YcMG54W4BQUFza5Pqa2t1fz585Wbm6uQkBCNGzdOq1evVnh4eLP3/fDDD1VQUKCpU6fe2B4BLjhTXad1e05KkmZQww8AbsvlHhZ3RQ8LWmPxxqP6XfYxJceHa/2PRrW49wcA0DbapYcF8CYX6+1avfWEJGnG2CTCCgC4MQILOq0395zU+QsNiu8RrIzbqOEHAHdGYEGnZHcYrdjcdCvzo6P7yt+P3woA4M74Uxqd0saDpTpx9oLCggP0QAotyQDg7ggs6JSWXzq78tAdfdQ1iNvoAcDdEVjQ6ezOP69d+ecV6OerKWmJVo8DAGgBAgs6nctnV743LFZRobavWQ0AcAcEFnQq+WdrtOGLpp8sPm0sRXEA4CkILOhUVuTkyRjpX26J1M3R3aweBwDQQgQWdBrna+r1+q5CSdJ0avgBwKMQWNBprNmWr9oGhwbFhSotqafV4wAAXEBgQadQ22DXqks1/NOp4QcAj0NgQaewfm+RzlTXKy48WOMG97J6HACAiwgs8HoOh9GyS7cyZ45OVAA1/ADgcfiTG17v70fK9OXpGnWz+euHI/tYPQ4AoBUILPB6Sz9pOrvy76l9FEINPwB4JAILvNpnheXanndO/r4+yhzV1+pxAACtRGCBV7t87cp9Q2MVE0YNPwB4KgILvFbhuQt6b3+xpKZbmQEAnovAAq/18pY8OYw0tn+Ebu0VavU4AIAbQGCBV6q40KC1O5tq+GdQww8AHo/AAq/06o58Xai3a0BMN425KcLqcQAAN4jAAq9T12jXK1tOSGo6u0INPwB4PgILvM47+06prKpOMaE2fXdIrNXjAADaAIEFXsWY5jX8gf58xAHAG/CnObzKpqOndbS0WiFB/nowlRp+APAWBBZ4lctnV344Il6htgCLpwEAtBUCC7zGgaIKbTl+Vn6+PsocQw0/AHgTAgu8xvJLZ1fuHdxLceHBFk8DAGhLBBZ4hVPlF/WXz6nhBwBvRWCBV1i5JU92h1FaUk8N7h1m9TgAgDZGYIHHq6xt0J92UMMPAN6MwAKP99qOAlXXNap/VIi+cXOk1eMAANoBgQUerb7RoZdzTkhqunbF15cafgDwRgQWeLS/7j+lkspaRYQEafwwavgBwFsRWOCxjDFa+kmepKYa/iB/P4snAgC0FwILPNaW42d1qLhSwQF+mkQNPwB4NQILPNbSS0VxE0fEK7xLoMXTAADaE4EFHulQcaU+OXpavj7S1NHU8AOAtyOwwCMt39x07co9g3qpT88uFk8DAGhvBBZ4nJKKWr3zWZEkadpYzq4AQGdAYIHHeeXTE2qwG41M7KFhfbpbPQ4AoAMQWOBRqusa9er2fEnSdGr4AaDTILDAo6zdWaiq2kYlRXTVXQOirB4HANBBCCzwGI12h17OabrYdho1/ADQqRBY4DHeO1CiovKL6tk1UP92e5zV4wAAOhCBBR6hqYb/S0nS5LRE2QKo4QeAzoTAAo+wLfecDhRVKsjfVw+nJVg9DgCggxFY4BGWXarhfyClt3p0pYYfADobAgvc3rHSKn10uEw+PtKjY7iVGQA6IwIL3N7lGv7vDIxW34iuFk8DALACgQVurayqVm/vbarhn0FRHAB0WgQWuLXVW/NVb3fo9j7hGp7Qw+pxAAAWIbDAbV2ob9TqbU01/JxdAYDOjcACt/Xm7pMqv9CghJ5d9O2BMVaPAwCwEIEFbsnuMM6LbaeN6Ss/avgBoFMjsMAt/e2LEhWcu6DuXQL0/eHxVo8DALAYgQVuxxijlz5pKop7+I4EBQdSww8AnR2BBW5nd/557SssV6C/rx5OS7R6HACAGyCwwO0svXR25f7b4xTZLcjiaQAA7oDAAreSe7paGw+VSqKGHwDwFQIL3MqKnDwZI6XfGqWbokKsHgcA4CYILHAbZ6vr9Obuk5Kk6WM5uwIA+AqBBW5j9bZ81TU6lNw7TCP7UsMPAPgKgQVuobbBrj9sbarhn35nknx8KIoDAHyFwAK3sG7PSZ2rqVfv7sG6+zZq+AEAzRFYYDnHP9TwPzqmr/z9+FgCAJpr1TfD888/r8TERNlsNqWmpmrHjh3XXNvQ0KCnnnpK/fr1k81mU3JysjZs2HDFuqKiIj300EPq2bOngoODNXjwYO3atas148HDfHioVHlnahRq89cPUqjhBwBcyeXAsnbtWs2ZM0cLFy7Unj17lJycrIyMDJWVlV11/fz58/XSSy/pueee08GDBzVz5kxNmDBBe/fuda45f/68Ro8erYCAAL3//vs6ePCgfvvb36p79+6t3zN4jGWbm4riHrojQV2D/C2eBgDgjnyMMcaVF6SmpmrEiBH6/e9/L0lyOByKj4/Xj3/8Y82dO/eK9bGxsfrlL3+pWbNmOR+7//77FRwcrDVr1kiS5s6dqy1btmjz5s2t3pHKykqFhYWpoqJCoaGhrX4fdKw9Bef1b//3qQL8fLTl599SVKjN6pEAAB2opd/fLp1hqa+v1+7du5Wenv7VG/j6Kj09XVu3br3qa+rq6mSzNf8SCg4OVk5OjvPf33nnHaWkpOiBBx5QVFSUhg0bpmXLll13lrq6OlVWVjbb4HmWXzq78r2hcYQVAMA1uRRYzpw5I7vdrujo6GaPR0dHq6Sk5KqvycjI0OLFi3Xs2DE5HA5t3LhRb731loqLi51rcnNz9cILL6h///764IMP9Nhjj+knP/mJVq1adc1ZsrKyFBYW5tzi47n2wdPkn63RhgNNn5vpd1IUBwC4tna/HWPJkiXq37+/BgwYoMDAQM2ePVuZmZny9f3ql3Y4HLr99tv161//WsOGDdOMGTM0ffp0vfjii9d833nz5qmiosK5FRYWtveuoI29nJMnh5G+eUukbo7uZvU4AAA35lJgiYiIkJ+fn0pLS5s9XlpaqpiYq3dnREZGav369aqpqVF+fr4OHz6skJAQJSV99X/UvXr10sCBA5u97tZbb1VBQcE1ZwkKClJoaGizDZ7jfE29Xt/VVMM/gxp+AMDXcCmwBAYGavjw4crOznY+5nA4lJ2drbS0tOu+1mazKS4uTo2NjVq3bp3Gjx/vfG706NE6cuRIs/VHjx5VQkKCK+PBg7y6PV8XG+y6LTZUaf16Wj0OAMDNuXwP6Zw5czRlyhSlpKRo5MiRevbZZ1VTU6PMzExJ0uTJkxUXF6esrCxJ0vbt21VUVKShQ4eqqKhIixYtksPh0JNPPul8z//8z//UqFGj9Otf/1o/+MEPtGPHDi1dulRLly5to92EO6ltsOuVT5tq+GdQww8AaAGXA8vEiRN1+vRpLViwQCUlJRo6dKg2bNjgvBC3oKCg2fUptbW1mj9/vnJzcxUSEqJx48Zp9erVCg8Pd64ZMWKE3n77bc2bN09PPfWU+vbtq2effVaTJk268T2E2/nzviKdqa5TbJhN4wb3snocAIAHcLmHxV3Rw+IZHA6j7zz7iY6XVWv+vbdqGtevAECn1i49LMCN+vhomY6XVatbkL8mjuBWdABAyxBY0KGWftJUFPfvqX3UzRZg8TQAAE9BYEGH+fxkubblnpO/r48eGZ1o9TgAAA9CYEGHWbY5T5J0X3KseoUFWzwNAMCTEFjQIQrPXdB7+5t+HAMX2gIAXEVgQYdYueWE7A6jsf0jNDCWu7gAAK4hsKDdVVxo0Gs7m37MAmdXAACtQWBBu/vjjgJdqLfrluhuurN/hNXjAAA8EIEF7aq+0aGVW5outp1ODT8AoJUILGhX73x2SmVVdYoODdJ9ybFWjwMA8FAEFrQbY4yWXSqKe2RUXwX683EDALQO3yBoN58cO6MjpVXqGuinf0/tY/U4AAAPRmBBu7l8dmXiiD4KC6aGHwDQegQWtIsvTlUo5/gZ+fn6KJMafgDADSKwoF0sv1TDP25wL8X36GLxNAAAT0dgQZs7VX5Rf/nslCRp+ti+Fk8DAPAGBBa0uVc+PaFGh9EdST00pHe41eMAALwAgQVtqrK2QX/c3lTDP+NOavgBAG2DwII2tXZHoarrGnVTVIi+eXOU1eMAALwEgQVtpsHu0MuXa/jH9pWvLzX8AIC2QWBBm3lvf7GKK2oVERKk8UPjrB4HAOBFCCxoE8YYLXXW8CfIFuBn8UQAAG9CYEGb2PrlWX1xqlLBAX6alJpg9TgAAC9DYEGbWLq56ezKD1J6q3vXQIunAQB4GwILbtiRkip9fOS0fH2kqWMoigMAtD0CC27Y8ktnV+4eFKOEnl0tngYA4I0ILLghZZW1Wr+vSJI0fSxFcQCA9kFgwQ155dMTarAbjUjsrmF9uls9DgDASxFY0Go1dY1asy1fEmdXAADti8CCVnt9V6EqaxvVN6Kr0m+NtnocAIAXI7CgVRrtDq3Iaarhn0YNPwCgnRFY0CobvijRyfMX1aNroO6/vbfV4wAAvByBBS4zxmjZpRr+yWnU8AMA2h+BBS7bkXdOn52sUJC/rx6+gxp+AED7I7DAZcsuFcV9f3hv9QwJsngaAEBnQGCBS46XVevDQ2Xy8ZEepYYfANBBCCxwyYqcprMr3741WkmRIRZPAwDoLAgsaLHTVXVat6ephn/GnRTFAQA6DoEFLbZ66wnVNzo0rE+4hidQww8A6DgEFrTIxXq7/nCphn/G2CT5+FAUBwDoOAQWtMibuwtVfqFBfXp00Xdui7F6HABAJ0NgwdeyO4yW/0MNvx81/ACADkZgwdfaeLBE+WcvKLxLgL4/nBp+AEDHI7Dgay29VMP/8B0J6hLob/E0AIDOiMCC69qdf057CsoV6OeryWmJVo8DAOikCCy4rstnV/7t9jhFdqOGHwBgDQILrinvTI3+drBUUtPFtgAAWIXAgmtakZMrY6S7BkTppqhuVo8DAOjECCy4qrPVdXpj10lJ0nRq+AEAFiOw4KrWbCtQXaNDQ3qHKbVvD6vHAQB0cgQWXKG2wa4/bD0hSZpODT8AwA0QWHCFt/YU6WxNveLCg3XPIGr4AQDWI7CgGYfDaPnmpluZHx3TV/5+fEQAANbj2wjNZB8uU+6ZGoXa/PWDEfFWjwMAgCQCC/7JsktFcZPuSFBIEDX8AAD3QGCB096C89px4pwC/Hz0yKhEq8cBAMCJwAKn5ZvzJEnjh8YpOtRm8TQAAHyFwAJJUsHZC3r/QLGkpluZAQBwJwQWSJJe3pInh5G+cXOkbomhhh8A4F4ILFD5hXqt3VkoSZpBDT8AwA0RWKBXtxfoYoNdA3uFalS/nlaPAwDAFQgsnVxdo10rt5yQ1HR2hRp+AIA7IrB0cn/ee0pnquvUK8yme4f0snocAACuisDSiTkcRksv1fBnjk5UADX8AAA3xTdUJ7bp6GkdL6tWSJC/fjiyj9XjAABwTQSWTmzppRr+B0fGK9QWYPE0AABcW6sCy/PPP6/ExETZbDalpqZqx44d11zb0NCgp556Sv369ZPNZlNycrI2bNjQbM2iRYvk4+PTbBswYEBrRkML7T9Zoa25Z+Xv66PM0X2tHgcAgOtyObCsXbtWc+bM0cKFC7Vnzx4lJycrIyNDZWVlV10/f/58vfTSS3ruued08OBBzZw5UxMmTNDevXubrbvttttUXFzs3HJyclq3R2iRZZeuXfnukF6KDQ+2eBoAAK7P5cCyePFiTZ8+XZmZmRo4cKBefPFFdenSRS+//PJV169evVq/+MUvNG7cOCUlJemxxx7TuHHj9Nvf/rbZOn9/f8XExDi3iIiI1u0RvlZR+UX9dX9TDf80avgBAB7ApcBSX1+v3bt3Kz09/as38PVVenq6tm7detXX1NXVyWZr/oP0goODrziDcuzYMcXGxiopKUmTJk1SQUHBdWepq6tTZWVlsw0tszInT3aH0eibempQXJjV4wAA8LVcCixnzpyR3W5XdHR0s8ejo6NVUlJy1ddkZGRo8eLFOnbsmBwOhzZu3Ki33npLxcXFzjWpqal65ZVXtGHDBr3wwgvKy8vT2LFjVVVVdc1ZsrKyFBYW5tzi4+Nd2ZVOq+Jig/60oykM8kMOAQCeot3vElqyZIn69++vAQMGKDAwULNnz1ZmZqZ8fb/6pe+55x498MADGjJkiDIyMvTee++pvLxcr7/++jXfd968eaqoqHBuhYWF7b0rXuG1HQWqqbfrluhu+sbNkVaPAwBAi7gUWCIiIuTn56fS0tJmj5eWliomJuaqr4mMjNT69etVU1Oj/Px8HT58WCEhIUpKuvb/3YeHh+vmm2/W8ePHr7kmKChIoaGhzTZcX32jw1nDP21sX2r4AQAew6XAEhgYqOHDhys7O9v5mMPhUHZ2ttLS0q77WpvNpri4ODU2NmrdunUaP378NddWV1fryy+/VK9eVMW3pXc/P6WSylpFdQvSfUNjrR4HAIAWc/mvhObMmaNly5Zp1apVOnTokB577DHV1NQoMzNTkjR58mTNmzfPuX779u166623lJubq82bN+vuu++Ww+HQk08+6VzzX//1X9q0aZNOnDihTz/9VBMmTJCfn58efPDBNthFSJIxxlkU98joRAX5+1k8EQAALefv6gsmTpyo06dPa8GCBSopKdHQoUO1YcMG54W4BQUFza5Pqa2t1fz585Wbm6uQkBCNGzdOq1evVnh4uHPNyZMn9eCDD+rs2bOKjIzUmDFjtG3bNkVGco1FW8k5fkaHS6rUJdBPk0YmWD0OAAAu8THGGKuHaAuVlZUKCwtTRUUF17NcxcMrtmvzsTPKHJ2ohf96m9XjAAAgqeXf3/wsoU7gUHGlNh87I18faSo1/AAAD0Rg6QQu1/CPG9xL8T26WDwNAACuI7B4ueKKi3pn3ylJ0ow7KYoDAHgmAouXe+XTE2p0GKX27aEhvcOtHgcAgFYhsHixqtoG/XFbUw0/Z1cAAJ6MwOLF1u4sVFVdo/pFdtW/3BJl9TgAALQagcVLNdi/quGfPjZJvr7U8AMAPBeBxUu9t79YReUXFRESqO8Ni7N6HAAAbgiBxQsZY5y3Mk9JS5QtgBp+AIBnI7B4oa25Z3WgqFK2AF89dAc1/AAAz0dg8ULLLv2Qwx+kxKt710CLpwEA4MYRWLzM0dIq/f3Iafn4SI+OoYYfAOAdCCxeZvmla1fuvi1GCT27WjwNAABtg8DiRcoqa7V+b1MN/3SK4gAAXoTA4kVWbT2hertDKQnddXuf7laPAwBAmyGweImaukatuVTDz9kVAIC3IbB4iTd2FariYoP6RnRV+q3RVo8DAECbIrB4gUa7Qyu25ElqujPIjxp+AICXIbB4gQ++KFXhuYvq0TVQ99/e2+pxAABocwQWD2eM0dJPvpQkPXxHgoIDqeEHAHgfAouH23nivD47WaEgf189nEYNPwDAOxFYPNzSSzX89w/vrYiQIIunAQCgfRBYPNiXp6v14aFSavgBAF6PwOLBlm9uujMo/dZo9YsMsXgaAADaD4HFQ52prtO6PSclSTMoigMAeDkCi4f6w9Z81Tc6NDQ+XCkJ1PADALwbgcUDXay3a/XWE5Kazq74+FAUBwDwbgQWD/TmnpM6f6FB8T2ClXFbjNXjAADQ7ggsHsbuMFqxuelW5mljkqjhBwB0CgQWD7PxYKlOnL2gsOAAPZBCDT8AoHMgsHiYZZfOrjx8R4K6BPpbPA0AAB2DwOJBduef0+788wr089XkUdTwAwA6DwKLB1n2SVNR3IRhcYrqZrN4GgAAOg6BxUOcOFOjDw6WSJKmjaWGHwDQuRBYPMSKnDwZI31rQJT6R3ezehwAADoUgcUDnKup1xu7CyVJ08dSww8A6HwILB5gzbZ81TY4NDguTHck9bB6HAAAOhyBxc3VNtj1h0s1/NOp4QcAdFIEFje3fm+RzlTXKy48WOMGUcMPAOicCCxuzOEwzqK4qWP6yt+P/1wAgM6Jb0A39vcjZfrydI262fw1cUS81eMAAGAZAosbW/pJ09mVSakJCgmihh8A0HkRWNzUZ4Xl2p53Tv6+PnpkVKLV4wAAYCkCi5u6fO3KfUNjFRNGDT8AoHMjsLihwnMX9N7+YkkUxQEAIBFY3NLLW/LkMNLY/hG6tVeo1eMAAGA5AoubqbjQoLU7m2r4Z9zJ2RUAACQCi9t5dUe+LtTbNSCmm8bcFGH1OAAAuAUCixupa7TrlS0nJDWdXaGGHwCAJgQWN/LOvlMqq6pTTKhN3x0Sa/U4AAC4DQKLmzDmqxr+zNGJCvTnPw0AAJfxregmNh09raOl1QoJ8teDqX2sHgcAALdCYHETl8+u/HBEvEJtARZPAwCAeyGwuIEDRRXacvys/Hx9lDmmr9XjAADgdggsbmD5pbMr3x3SS3HhwRZPAwCA+yGwWOxU+UX95XNq+AEAuB4Ci8VWbsmT3WE0ql9PDYoLs3ocAADcEoHFQpW1DfrTjqYa/unU8AMAcE0EFgu9tqNA1XWN6h8Vom/eHGn1OAAAuC0Ci0XqGx16OeeEpKazK9TwAwBwbQQWi/x1/ymVVNYqsluQxg+lhh8AgOshsFjAGKOln+RJkh4Zlaggfz+LJwIAwL0RWCyw5fhZHSquVJdAP02ihh8AgK9FYLHA0ktFcT9IiVd4l0CLpwEAwP0RWDrYoeJKfXL0tHx9pEep4QcAoEUILB1s+eama1fuGdxL8T26WDwNAACeoVWB5fnnn1diYqJsNptSU1O1Y8eOa65taGjQU089pX79+slmsyk5OVkbNmy45vpnnnlGPj4+euKJJ1ozmlsrqajVO58VSZJmUMMPAECLuRxY1q5dqzlz5mjhwoXas2ePkpOTlZGRobKysquunz9/vl566SU999xzOnjwoGbOnKkJEyZo7969V6zduXOnXnrpJQ0ZMsT1PfEAr3x6Qg12o5F9eyg5PtzqcQAA8BguB5bFixdr+vTpyszM1MCBA/Xiiy+qS5cuevnll6+6fvXq1frFL36hcePGKSkpSY899pjGjRun3/72t83WVVdXa9KkSVq2bJm6d+/eur1xY9V1jXp1e74kzq4AAOAqlwJLfX29du/erfT09K/ewNdX6enp2rp161VfU1dXJ5vN1uyx4OBg5eTkNHts1qxZuvfee5u99/XU1dWpsrKy2ebO1u4sVFVto5Iiu+pbA6KsHgcAAI/iUmA5c+aM7Ha7oqOjmz0eHR2tkpKSq74mIyNDixcv1rFjx+RwOLRx40a99dZbKi4udq557bXXtGfPHmVlZbV4lqysLIWFhTm3+Ph4V3alQzXaHXo5p+li2+ljk+TrSw0/AACuaPe7hJYsWaL+/ftrwIABCgwM1OzZs5WZmSlf36ZfurCwUI8//rheffXVK87EXM+8efNUUVHh3AoLC9trF27YewdKVFR+UREhgZowLM7qcQAA8DguBZaIiAj5+fmptLS02eOlpaWKiYm56msiIyO1fv161dTUKD8/X4cPH1ZISIiSkpqu49i9e7fKysp0++23y9/fX/7+/tq0aZN+97vfyd/fX3a7/arvGxQUpNDQ0GabO2qq4f9SkjQ5LVG2AGr4AQBwlUuBJTAwUMOHD1d2drbzMYfDoezsbKWlpV33tTabTXFxcWpsbNS6des0fvx4SdJdd92l/fv3a9++fc4tJSVFkyZN0r59++Tn59lf8Ntyz+lAUaVsAb566I4Eq8cBAMAj+bv6gjlz5mjKlClKSUnRyJEj9eyzz6qmpkaZmZmSpMmTJysuLs55Pcr27dtVVFSkoUOHqqioSIsWLZLD4dCTTz4pSerWrZsGDRrU7Nfo2rWrevbsecXjnmjZpRr+B4bHq0dXavgBAGgNlwPLxIkTdfr0aS1YsEAlJSUaOnSoNmzY4LwQt6CgwHl9iiTV1tZq/vz5ys3NVUhIiMaNG6fVq1crPDy8zXbCXR0rrdJHh8vkQw0/AAA3xMcYY6weoi1UVlYqLCxMFRUVbnM9y8/f/FxrdxXq7tti9OLDw60eBwAAt9PS729+llA7Kauq1dt7m2r4p99JURwAADeCwNJO/vBpvurtDg1P6K7hCd7X3AsAQEcisLSDC/WNWr2tqYZ/OjX8AADcMAJLO3hj10lVXGxQYs8u+vbA6K9/AQAAuC4CSxuzO4yW5zTdyvzo2CT5UcMPAMANI7C0sQ++KFHhuYvq3iVA37+9t9XjAADgFQgsbcgYo5c+aTq78nBaooIDPbulFwAAd0FgaUO78s/rs8JyBfr7anIaNfwAALQVAksbWnrp7Mr9t/dWREiQxdMAAOA9CCxtJPd0tT481PRTrKeNpYYfAIC2RGBpIyty8mSMlH5rtPpFhlg9DgAAXoXA0gbOVtfpzd0nJUkzqOEHAKDNEVjawOpt+aprdCg5PlwjEqnhBwCgrRFYblBtg11/2NpUwz9jbJJ8fCiKAwCgrRFYbtC6PSd1rqZe8T2ClXEbNfwAALQHAssNcDiMlm/OkyQ9Orqv/P04nAAAtAe+YW/Ah4dKlXemRmHBAXogJd7qcQAA8FoElhuwbHNTUdxDd/RR1yB/i6cBAMB7EVhaaU/Bee08cV6Bfr6akpZo9TgAAHg1AksrLb90duV7w2IVFWqzeBoAALwbgaUV8s/WaMOBEknStLEUxQEA0N4ILK3wck6eHEb65i2Rujm6m9XjAADg9QgsLjpfU6/Xd12q4efsCgAAHYLA4qJXt+frYoNdt8WGKq1fT6vHAQCgUyCwuKC2wa5XPr1Uw38nNfwAAHQUAosL/ryvSGeq6xQbZtO4wb2sHgcAgE6DwNJCDofRsks1/FPH9FUANfwAAHQYvnVb6OOjZTpeVq1uQf6aOIIafgAAOhKBpYWWftJUFPfvqX3UzRZg8TQAAHQuBJYW+PxkubblnpO/r48eGZ1o9TgAAHQ6BJYWuHztyn3JseoVFmzxNAAAdD4Elq9ReO6C3ttfLIkafgAArEJg+Rort5yQ3WE0tn+EBsaGWj0OAACdEoHlOiouNui1nQWSpOmcXQEAwDL+Vg/gzkKC/PW/DyTrw0OlGts/wupxAADotAgs1+Hn66Nxg3vRagsAgMX4KyEAAOD2CCwAAMDtEVgAAIDbI7AAAAC3R2ABAABuj8ACAADcHoEFAAC4PQILAABwewQWAADg9ggsAADA7RFYAACA2yOwAAAAt0dgAQAAbs9rflqzMUaSVFlZafEkAACgpS5/b1/+Hr8WrwksVVVVkqT4+HiLJwEAAK6qqqpSWFjYNZ/3MV8XaTyEw+HQqVOn1K1bN/n4+Fg6S2VlpeLj41VYWKjQ0FBLZ/F0HMu2wXFsGxzHtsFxbDvecCyNMaqqqlJsbKx8fa99pYrXnGHx9fVV7969rR6jmdDQUI/9ALkbjmXb4Di2DY5j2+A4th1PP5bXO7NyGRfdAgAAt0dgAQAAbo/A0g6CgoK0cOFCBQUFWT2Kx+NYtg2OY9vgOLYNjmPb6UzH0msuugUAAN6LMywAAMDtEVgAAIDbI7AAAAC3R2ABAABuj8DSQllZWRoxYoS6deumqKgofe9739ORI0earamtrdWsWbPUs2dPhYSE6P7771dpaWmzNQUFBbr33nvVpUsXRUVF6Wc/+5kaGxs7clcs98ILL2jIkCHOoqO0tDS9//77zuc5jq575pln5OPjoyeeeML5GMexZRYtWiQfH59m24ABA5zPcxxbrqioSA899JB69uyp4OBgDR48WLt27XI+b4zRggUL1KtXLwUHBys9PV3Hjh1r9h7nzp3TpEmTFBoaqvDwcD366KOqrq7u6F2xVGJi4hWfSR8fH82aNUtSJ/5MGrRIRkaGWblypTlw4IDZt2+fGTdunOnTp4+prq52rpk5c6aJj4832dnZZteuXeaOO+4wo0aNcj7f2NhoBg0aZNLT083evXvNe++9ZyIiIsy8efOs2CXLvPPOO+avf/2rOXr0qDly5Ij5xS9+YQICAsyBAweMMRxHV+3YscMkJiaaIUOGmMcff9z5OMexZRYuXGhuu+02U1xc7NxOnz7tfJ7j2DLnzp0zCQkJ5pFHHjHbt283ubm55oMPPjDHjx93rnnmmWdMWFiYWb9+vfnss8/MfffdZ/r27WsuXrzoXHP33Xeb5ORks23bNrN582Zz0003mQcffNCKXbJMWVlZs8/jxo0bjSTz97//3RjTeT+TBJZWKisrM5LMpk2bjDHGlJeXm4CAAPPGG2841xw6dMhIMlu3bjXGGPPee+8ZX19fU1JS4lzzwgsvmNDQUFNXV9exO+BmunfvbpYvX85xdFFVVZXp37+/2bhxo/nGN77hDCwcx5ZbuHChSU5OvupzHMeW+/nPf27GjBlzzecdDoeJiYkxv/nNb5yPlZeXm6CgIPOnP/3JGGPMwYMHjSSzc+dO55r333/f+Pj4mKKiovYb3s09/vjjpl+/fsbhcHTqzyR/JdRKFRUVkqQePXpIknbv3q2Ghgalp6c71wwYMEB9+vTR1q1bJUlbt27V4MGDFR0d7VyTkZGhyspKffHFFx04vfuw2+167bXXVFNTo7S0NI6ji2bNmqV777232fGS+Dy66tixY4qNjVVSUpImTZqkgoICSRxHV7zzzjtKSUnRAw88oKioKA0bNkzLli1zPp+Xl6eSkpJmxzIsLEypqanNjmV4eLhSUlKca9LT0+Xr66vt27d33M64kfr6eq1Zs0ZTp06Vj49Pp/5MElhaweFw6IknntDo0aM1aNAgSVJJSYkCAwMVHh7ebG10dLRKSkqca/7xA3T5+cvPdSb79+9XSEiIgoKCNHPmTL399tsaOHAgx9EFr732mvbs2aOsrKwrnuM4tlxqaqpeeeUVbdiwQS+88ILy8vI0duxYVVVVcRxdkJubqxdeeEH9+/fXBx98oMcee0w/+clPtGrVKklfHYurHat/PJZRUVHNnvf391ePHj061bH8R+vXr1d5ebkeeeQRSZ3797bX/LTmjjRr1iwdOHBAOTk5Vo/isW655Rbt27dPFRUVevPNNzVlyhRt2rTJ6rE8RmFhoR5//HFt3LhRNpvN6nE82j333OP85yFDhig1NVUJCQl6/fXXFRwcbOFknsXhcCglJUW//vWvJUnDhg3TgQMH9OKLL2rKlCkWT+e5VqxYoXvuuUexsbFWj2I5zrC4aPbs2Xr33Xf197//Xb1793Y+HhMTo/r6epWXlzdbX1paqpiYGOeaf76S+/K/X17TWQQGBuqmm27S8OHDlZWVpeTkZC1ZsoTj2EK7d+9WWVmZbr/9dvn7+8vf31+bNm3S7373O/n7+ys6Oprj2Erh4eG6+eabdfz4cT6PLujVq5cGDhzY7LFbb73V+ddrl4/F1Y7VPx7LsrKyZs83Njbq3LlznepYXpafn68PP/xQ06ZNcz7WmT+TBJYWMsZo9uzZevvtt/XRRx+pb9++zZ4fPny4AgIClJ2d7XzsyJEjKigoUFpamiQpLS1N+/fvb/YbcuPGjQoNDb3iN3pn43A4VFdXx3Fsobvuukv79+/Xvn37nFtKSoomTZrk/GeOY+tUV1fryy+/VK9evfg8umD06NFXVD0cPXpUCQkJkqS+ffsqJiam2bGsrKzU9u3bmx3L8vJy7d6927nmo48+ksPhUGpqagfshXtZuXKloqKidO+99zof69SfSauv+vUUjz32mAkLCzMff/xxs9vNLly44Fwzc+ZM06dPH/PRRx+ZXbt2mbS0NJOWluZ8/vKtZt/5znfMvn37zIYNG0xkZKTH32rmqrlz55pNmzaZvLw88/nnn5u5c+caHx8f87e//c0Yw3FsrX+8S8gYjmNL/fSnPzUff/yxycvLM1u2bDHp6ekmIiLClJWVGWM4ji21Y8cO4+/vb55++mlz7Ngx8+qrr5ouXbqYNWvWONc888wzJjw83Pz5z382n3/+uRk/fvxVb2seNmyY2b59u8nJyTH9+/fvdLc1G2OM3W43ffr0MT//+c+veK6zfiYJLC0k6arbypUrnWsuXrxofvSjH5nu3bubLl26mAkTJpji4uJm73PixAlzzz33mODgYBMREWF++tOfmoaGhg7eG2tNnTrVJCQkmMDAQBMZGWnuuusuZ1gxhuPYWv8cWDiOLTNx4kTTq1cvExgYaOLi4szEiRObdYdwHFvuL3/5ixk0aJAJCgoyAwYMMEuXLm32vMPhML/61a9MdHS0CQoKMnfddZc5cuRIszVnz541Dz74oAkJCTGhoaEmMzPTVFVVdeRuuIUPPvjASLri+BjTeT+TPsYYY+EJHgAAgK/FNSwAAMDtEVgAAIDbI7AAAAC3R2ABAABuj8ACAADcHoEFAAC4PQILAABwewQWAADg9ggsAADA7RFYAACA2yOwAAAAt0dgAQAAbu//A6SFI2mNUQxoAAAAAElFTkSuQmCC", + "text/plain": [ + "
" + ] + }, + "metadata": {}, + "output_type": "display_data" + } + ], + "source": [ + "results[results['classification/accuracy'].notnull()]['classification/accuracy'].plot()" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [] + } + ], + "metadata": { + "kernelspec": { + "display_name": "Python 3", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.9.16" + }, + "orig_nbformat": 4 + }, + "nbformat": 4, + "nbformat_minor": 2 +} diff --git a/AI4Forensics/IP_Identifier_Fine_Tuning/code/file_manager.ipynb b/AI4Forensics/IP_Identifier_Fine_Tuning/code/file_manager.ipynb new file mode 100644 index 0000000..7e75e8c --- /dev/null +++ b/AI4Forensics/IP_Identifier_Fine_Tuning/code/file_manager.ipynb @@ -0,0 +1,253 @@ +{ + "cells": [ + { + "cell_type": "code", + "execution_count": 1, + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + " JSON: {\n", + " \"object\": \"list\",\n", + " \"data\": [\n", + " {\n", + " \"object\": \"file\",\n", + " \"id\": \"file-NPzEnJeO6kohiGDbcQoxNWIQ\",\n", + " \"purpose\": \"fine-tune\",\n", + " \"filename\": \"dataset_train.jsonl\",\n", + " \"bytes\": 21860,\n", + " \"created_at\": 1692102727,\n", + " \"status\": \"processed\",\n", + " \"status_details\": null\n", + " },\n", + " {\n", + " \"object\": \"file\",\n", + " \"id\": \"file-ngHRotibSy82Mr55IjsPP2uK\",\n", + " \"purpose\": \"fine-tune-results\",\n", + " \"filename\": \"compiled_results.csv\",\n", + " \"bytes\": 31134,\n", + " \"created_at\": 1692016747,\n", + " \"status\": \"processed\",\n", + " \"status_details\": null\n", + " },\n", + " {\n", + " \"object\": \"file\",\n", + " \"id\": \"file-1kZdBPJtBAHOehkHuhc8z5N1\",\n", + " \"purpose\": \"fine-tune\",\n", + " \"filename\": \"file\",\n", + " \"bytes\": 21860,\n", + " \"created_at\": 1691961832,\n", + " \"status\": \"processed\",\n", + " \"status_details\": null\n", + " },\n", + " {\n", + " \"object\": \"file\",\n", + " \"id\": \"file-u1WXeeJsNuLCyJCsZA6RVvyL\",\n", + " \"purpose\": \"fine-tune\",\n", + " \"filename\": \"dataset_test.jsonl\",\n", + " \"bytes\": 4608,\n", + " \"created_at\": 1692283892,\n", + " \"status\": \"processed\",\n", + " \"status_details\": null\n", + " },\n", + " {\n", + " \"object\": \"file\",\n", + " \"id\": \"file-vBNjX2uPT56mTb17Zb4trVn6\",\n", + " \"purpose\": \"fine-tune-results\",\n", + " \"filename\": \"compiled_results.csv\",\n", + " \"bytes\": 40617,\n", + " \"created_at\": 1692317059,\n", + " \"status\": \"processed\",\n", + " \"status_details\": null\n", + " },\n", + " {\n", + " \"object\": \"file\",\n", + " \"id\": \"file-DskZVtrrakjLZmNA3kS7jVxp\",\n", + " \"purpose\": \"fine-tune\",\n", + " \"filename\": \"./dataset_train.jsonl\",\n", + " \"bytes\": 21860,\n", + " \"created_at\": 1692316870,\n", + " \"status\": \"processed\",\n", + " \"status_details\": null\n", + " },\n", + " {\n", + " \"object\": \"file\",\n", + " \"id\": \"file-tqxJwOgJsarbldDBKnojj4K0\",\n", + " \"purpose\": \"fine-tune\",\n", + " \"filename\": \"./dataset_test.jsonl\",\n", + " \"bytes\": 4608,\n", + " \"created_at\": 1692316875,\n", + " \"status\": \"processed\",\n", + " \"status_details\": null\n", + " }\n", + " ]\n", + "}" + ] + }, + "execution_count": 1, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "import os\n", + "import openai\n", + "\n", + "openai.api_key = \"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\"\n", + "openai.File.list()" + ] + }, + { + "cell_type": "code", + "execution_count": 2, + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + " JSON: {\n", + " \"object\": \"list\",\n", + " \"data\": [\n", + " {\n", + " \"object\": \"fine-tune\",\n", + " \"id\": \"ft-3DA3kW5k0BaHoWqwjJUB6Xwf\",\n", + " \"hyperparams\": {\n", + " \"n_epochs\": 4,\n", + " \"batch_size\": 1,\n", + " \"prompt_loss_weight\": 0.01,\n", + " \"learning_rate_multiplier\": 0.1\n", + " },\n", + " \"organization_id\": \"org-c8msxWICKfgPglwGB7XFRGy8\",\n", + " \"model\": \"ada\",\n", + " \"training_files\": [\n", + " {\n", + " \"object\": \"file\",\n", + " \"id\": \"file-1kZdBPJtBAHOehkHuhc8z5N1\",\n", + " \"purpose\": \"fine-tune\",\n", + " \"filename\": \"file\",\n", + " \"bytes\": 21860,\n", + " \"created_at\": 1691961832,\n", + " \"status\": \"processed\",\n", + " \"status_details\": null\n", + " }\n", + " ],\n", + " \"validation_files\": [],\n", + " \"result_files\": [\n", + " {\n", + " \"object\": \"file\",\n", + " \"id\": \"file-ngHRotibSy82Mr55IjsPP2uK\",\n", + " \"purpose\": \"fine-tune-results\",\n", + " \"filename\": \"compiled_results.csv\",\n", + " \"bytes\": 31134,\n", + " \"created_at\": 1692016747,\n", + " \"status\": \"processed\",\n", + " \"status_details\": null\n", + " }\n", + " ],\n", + " \"created_at\": 1692015392,\n", + " \"updated_at\": 1692016748,\n", + " \"status\": \"succeeded\",\n", + " \"fine_tuned_model\": \"ada:ft-university-of-baltimore-2023-08-14-12-39-06\"\n", + " },\n", + " {\n", + " \"object\": \"fine-tune\",\n", + " \"id\": \"ft-UjMMhr7J7K5pBVJMH7lTO5Oe\",\n", + " \"hyperparams\": {\n", + " \"n_epochs\": 4,\n", + " \"batch_size\": 1,\n", + " \"prompt_loss_weight\": 0.01,\n", + " \"learning_rate_multiplier\": 0.1,\n", + " \"classification_positive_class\": \" positive\",\n", + " \"compute_classification_metrics\": true\n", + " },\n", + " \"organization_id\": \"org-c8msxWICKfgPglwGB7XFRGy8\",\n", + " \"model\": \"ada\",\n", + " \"training_files\": [\n", + " {\n", + " \"object\": \"file\",\n", + " \"id\": \"file-DskZVtrrakjLZmNA3kS7jVxp\",\n", + " \"purpose\": \"fine-tune\",\n", + " \"filename\": \"./dataset_train.jsonl\",\n", + " \"bytes\": 21860,\n", + " \"created_at\": 1692316870,\n", + " \"status\": \"processed\",\n", + " \"status_details\": null\n", + " }\n", + " ],\n", + " \"validation_files\": [\n", + " {\n", + " \"object\": \"file\",\n", + " \"id\": \"file-tqxJwOgJsarbldDBKnojj4K0\",\n", + " \"purpose\": \"fine-tune\",\n", + " \"filename\": \"./dataset_test.jsonl\",\n", + " \"bytes\": 4608,\n", + " \"created_at\": 1692316875,\n", + " \"status\": \"processed\",\n", + " \"status_details\": null\n", + " }\n", + " ],\n", + " \"result_files\": [\n", + " {\n", + " \"object\": \"file\",\n", + " \"id\": \"file-vBNjX2uPT56mTb17Zb4trVn6\",\n", + " \"purpose\": \"fine-tune-results\",\n", + " \"filename\": \"compiled_results.csv\",\n", + " \"bytes\": 40617,\n", + " \"created_at\": 1692317059,\n", + " \"status\": \"processed\",\n", + " \"status_details\": null\n", + " }\n", + " ],\n", + " \"created_at\": 1692316875,\n", + " \"updated_at\": 1692317060,\n", + " \"status\": \"succeeded\",\n", + " \"fine_tuned_model\": \"ada:ft-university-of-baltimore-2023-08-18-00-04-18\"\n", + " }\n", + " ],\n", + " \"next_starting_after\": null\n", + "}" + ] + }, + "execution_count": 2, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "openai.FineTune.list()\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "openai.FineTune.list()" + ] + } + ], + "metadata": { + "kernelspec": { + "display_name": "Python 3", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.9.16" + }, + "orig_nbformat": 4 + }, + "nbformat": 4, + "nbformat_minor": 2 +} diff --git a/AI4Forensics/IP_Identifier_Fine_Tuning/code/result.csv b/AI4Forensics/IP_Identifier_Fine_Tuning/code/result.csv new file mode 100644 index 0000000..853c547 --- /dev/null +++ b/AI4Forensics/IP_Identifier_Fine_Tuning/code/result.csv @@ -0,0 +1,735 @@ +step,elapsed_tokens,elapsed_examples,training_loss,training_sequence_accuracy,training_token_accuracy,validation_loss,validation_sequence_accuracy,validation_token_accuracy,classification/accuracy,classification/precision,classification/recall,classification/auroc,classification/auprc,classification/f1.0 +1,41,1,0.36131611532358515,0.0,0.0,0.4872071744380082,0.0,0.0,,,,,, +2,74,2,0.3700738534195625,0.0,0.0,,,,,,,,, +3,107,3,0.22843073428540267,0.0,0.0,,,,,,,,, +4,148,4,0.18925983548845396,0.0,0.0,,,,,,,,, +5,173,5,0.06851752833268013,1.0,1.0,,,,,,,,, +6,262,6,0.02713874108659432,1.0,1.0,,,,,,,,, +7,375,7,0.05557006998931683,1.0,1.0,,,,,,,,, +8,416,8,0.037689568018540745,1.0,1.0,,,,,,,,, +9,441,9,0.03327601474069525,1.0,1.0,0.031220039141662707,1.0,1.0,,,,,, +10,506,10,0.031220039141662707,1.0,1.0,,,,,,,,, +11,539,11,0.22557505111595672,0.0,0.0,,,,,,,,, +12,620,12,0.0694835970180284,0.0,0.0,,,,,,,,, +13,661,13,0.0693355020871386,0.0,0.0,,,,,,,,, +14,694,14,0.045931317708786994,0.0,0.0,,,,,,,,, +15,735,15,0.028463621933449756,1.0,1.0,,,,,,,,, +16,760,16,0.020192973992889166,1.0,1.0,,,,,,,,, +17,785,17,0.017002813986327962,1.0,1.0,0.028524243063839095,1.0,1.0,,,,,, +18,874,18,0.026500673530919647,1.0,1.0,,,,,,,,, +19,899,19,0.022513926106815536,1.0,1.0,,,,,,,,, +20,1020,20,0.05054291217876016,1.0,1.0,,,,,,,,, +21,1053,21,0.17471788834329344,0.0,0.0,,,,,,,,, +22,1094,22,0.02836188224214129,1.0,1.0,,,,,,,,, +23,1135,23,0.034800293915439394,1.0,1.0,,,,,,,,, +24,1248,24,0.0544320389855836,1.0,1.0,,,,,,,,, +25,1281,25,0.03246741641443805,1.0,1.0,0.032951377297674,1.0,1.0,,,,,, +26,1322,26,0.03000724712666124,1.0,1.0,,,,,,,,, +27,1347,27,0.021077466012599567,1.0,1.0,,,,,,,,, +28,1372,28,0.02064567490713671,1.0,1.0,,,,,,,,, +29,1437,29,0.05725409627833869,0.0,0.0,,,,,,,,, +30,1470,30,0.016648737896612146,1.0,1.0,,,,,,,,, +31,1495,31,0.14679759012768046,0.0,0.0,,,,,,,,, +32,1528,32,0.020843235227948755,1.0,1.0,,,,,,,,, +33,1561,33,0.03276810964700416,1.0,1.0,0.04024752794350206,0.0,0.0,,,,,, +34,1626,34,0.040496221549910845,0.0,0.0,,,,,,,,, +35,1651,35,0.04781833109387662,0.0,0.0,,,,,,,,, +36,1684,36,0.030358177107700612,1.0,1.0,,,,,,,,, +37,1717,37,0.031779489604959965,1.0,1.0,,,,,,,,, +38,1830,38,0.051204410766845926,1.0,1.0,,,,,,,,, +39,1855,39,0.012860343157080935,1.0,1.0,,,,,,,,, +40,1880,40,0.014500696788697193,1.0,1.0,,,,,,,,, +41,1913,41,0.013097912614466625,1.0,1.0,0.01010702661355026,1.0,1.0,,,,,, +42,1954,42,0.017973546807654204,1.0,1.0,,,,,,,,, +43,1987,43,0.012875115118658869,1.0,1.0,,,,,,,,, +44,2012,44,0.008627477259433362,1.0,1.0,,,,,,,,, +45,2125,45,0.050168912711680086,1.0,1.0,,,,,,,,, +46,2166,46,0.026348685119068252,1.0,1.0,,,,,,,,, +47,2207,47,0.018747832050081344,1.0,1.0,,,,,,,,, +48,2248,48,0.02532638164306991,1.0,1.0,,,,,,,,, +49,2289,49,0.016451444947510028,1.0,1.0,0.2877718317729523,0.0,0.0,,,,,, +50,2330,50,0.02226830317033455,1.0,1.0,,,,,,,,, +51,2363,51,0.3006473934961832,0.0,0.0,,,,,,,,, +52,2396,52,0.00395260537712602,1.0,1.0,,,,,,,,, +53,2461,53,0.02284509134817199,1.0,1.0,,,,,,,,, +54,2542,54,0.026175346902920865,1.0,1.0,,,,,,,,, +55,2567,55,0.010952326249098403,1.0,1.0,,,,,,,,, +56,2592,56,0.014134095167488947,1.0,1.0,,,,,,,,, +57,2697,57,0.028973295194849088,1.0,1.0,0.01551858661980077,1.0,1.0,,,,,, +58,2722,58,0.014299217964702012,1.0,1.0,,,,,,,,, +59,2747,59,0.00834741592252006,1.0,1.0,,,,,,,,, +60,2788,60,0.02308588253887138,1.0,1.0,,,,,,,,, +61,2813,61,0.00906841204026326,1.0,1.0,,,,,,,,, +62,2934,62,0.046945270747467786,1.0,1.0,,,,,,,,, +63,2975,63,0.022567503625876274,1.0,1.0,,,,,,,,, +64,3008,64,0.007562187540606828,1.0,1.0,,,,,,,,, +65,3041,65,0.007075891750610026,1.0,1.0,0.11367509979154419,0.0,0.0,,,,,, +66,3106,66,0.026230665613984454,1.0,1.0,,,,,,,,, +67,3147,67,0.020585892829170917,1.0,1.0,,,,,,,,, +68,3172,68,0.007856541990040567,1.0,1.0,,,,,,,,, +69,3213,69,0.020074003617279235,1.0,1.0,,,,,,,,, +70,3334,70,0.0462450483632371,1.0,1.0,,,,,,,,, +71,3359,71,0.007845374103732561,1.0,1.0,,,,,,,,, +72,3400,72,0.01965699916551239,1.0,1.0,,,,,,,,, +73,3441,73,0.018607642447226683,1.0,1.0,0.00570118249716567,1.0,1.0,,,,,, +74,3474,74,0.00448214579580963,1.0,1.0,,,,,,,,, +75,3539,75,0.020936313641382185,1.0,1.0,,,,,,,,, +76,3572,76,0.1889746366855661,0.0,0.0,,,,,,,,, +77,3613,77,0.017631995898351304,1.0,1.0,,,,,,,,, +78,3654,78,0.015714159191644286,1.0,1.0,,,,,,,,, +79,3679,79,0.008651148353856115,1.0,1.0,,,,,,,,, +80,3720,80,0.020361671341634066,1.0,1.0,,,,,,,,, +81,3785,81,0.022455738474429835,1.0,1.0,0.022708754392936042,1.0,1.0,,,,,, +82,3826,82,0.022248722614873262,1.0,1.0,,,,,,,,, +83,3867,83,0.016634664889323178,1.0,1.0,,,,,,,,, +84,3892,84,0.015879148010002002,1.0,1.0,,,,,,,,, +85,3981,85,0.02129320362886078,1.0,1.0,,,,,,,,, +86,4022,86,0.013185540407779629,1.0,1.0,,,,,,,,, +87,4055,87,0.010587486446347612,1.0,1.0,,,,,,,,, +88,4080,88,0.02218080692789954,1.0,1.0,,,,,,,,, +89,4121,89,0.016163602579075814,1.0,1.0,0.018447448047036234,1.0,1.0,,,,,, +90,4210,90,0.018967157456083277,1.0,1.0,,,,,,,,, +91,4243,91,0.003596439868570087,1.0,1.0,,,,,,,,, +92,4316,92,0.10245768553965415,0.0,0.0,,,,,,,,, +93,4357,93,0.020223705363583578,1.0,1.0,,,,,,,,, +94,4390,94,0.008596739505919687,1.0,1.0,,,,,,,,, +95,4415,95,0.0059946262955782,1.0,1.0,,,,,,,,, +96,4448,96,0.02165651834432538,1.0,1.0,,,,,,,,, +97,4473,97,0.006453505932343735,1.0,1.0,0.004207050470713512,1.0,1.0,,,,,, +98,4594,98,0.046886877945600036,1.0,1.0,,,,,,,,, +99,4627,99,0.11959624585262645,0.0,0.0,,,,,,,,, +100,4652,100,0.009025092869123908,1.0,1.0,,,,,,,,, +101,4677,101,0.01159204928218969,1.0,1.0,,,,,,,,, +102,4718,102,0.021466303803201303,1.0,1.0,,,,,,,,, +103,4743,103,0.024004962817707566,1.0,1.0,,,,,,,,, +104,4816,104,0.04579475395743631,0.0,0.0,,,,,,,,, +105,4849,105,0.018203865305422368,1.0,1.0,0.04196793750077632,0.0,0.0,,,,,, +106,4890,106,0.017205431483860594,1.0,1.0,,,,,,,,, +107,4923,107,0.023016606093224257,1.0,1.0,,,,,,,,, +108,4996,108,0.03363417840498894,1.0,1.0,,,,,,,,, +109,5029,109,0.01055616332631871,1.0,1.0,,,,,,,,, +110,5070,110,0.010952473264303991,1.0,1.0,,,,,,,,, +111,5095,111,0.008982319584368573,1.0,1.0,,,,,,,,, +112,5120,112,0.00998764863734929,1.0,1.0,,,,,,,,, +113,5145,113,0.0102364023392632,1.0,1.0,0.1434572506475815,0.0,0.0,,,,,, +114,5186,114,0.010304133885405464,1.0,1.0,,,,,,,,, +115,5219,115,0.007398418430739184,1.0,1.0,,,,,,,,, +116,5252,116,0.160425701745653,0.0,0.0,,,,,,,,, +117,5341,117,0.01727596476598013,1.0,1.0,,,,,,,,, +118,5366,118,0.0039912293913948816,1.0,1.0,,,,,,,,, +119,5407,119,0.01271170173792052,1.0,1.0,,,,,,,,, +120,5520,120,0.04855149958462009,1.0,1.0,,,,,,,,, +121,5553,121,0.016561159628035966,1.0,1.0,0.07796625686372863,0.0,0.0,,,,,, +122,5586,122,0.03658135385616333,1.0,1.0,,,,,,,,, +123,5627,123,0.010930833159567555,1.0,1.0,,,,,,,,, +124,5668,124,0.011949884182671667,1.0,1.0,,,,,,,,, +125,5701,125,0.016956832591095008,1.0,1.0,,,,,,,,, +126,5726,126,0.022571875492479496,1.0,1.0,,,,,,,,, +127,5759,127,0.012888053933593255,1.0,1.0,,,,,,,,, +128,5792,128,0.005133158288572304,1.0,1.0,,,,,,,,, +129,5881,129,0.01916009849995052,1.0,1.0,0.013847733811394394,1.0,1.0,,,,,, +130,5906,130,0.012011411931428788,1.0,1.0,,,,,,,,, +131,5931,131,0.007971257554709154,1.0,1.0,,,,,,,,, +132,5964,132,0.08846963041842174,0.0,0.0,,,,,,,,, +133,6189,133,0.033074644572998295,1.0,1.0,,,,,,,,, +134,6310,134,0.04549703621705217,1.0,1.0,,,,,,,,, +135,6343,135,0.009617989432827016,1.0,1.0,,,,,,,,, +136,6384,136,0.014371102429315217,1.0,1.0,,,,,,,,, +137,6417,137,0.014548937769359328,1.0,1.0,0.00995008055856791,1.0,1.0,,,,,, +138,6538,138,0.04630857212240517,1.0,1.0,,,,,,,,, +139,6667,139,0.04324543540203195,1.0,1.0,,,,,,,,, +140,6708,140,0.010653898217366078,1.0,1.0,,,,,,,,, +141,6733,141,0.006511889914133159,1.0,1.0,,,,,,,,, +142,6774,142,0.011161583856330252,1.0,1.0,,,,,,,,, +143,6807,143,0.012623354419356475,1.0,1.0,,,,,,,,, +144,6856,144,0.02774330338124855,1.0,1.0,,,,,,,,, +145,6945,145,0.017150625479364613,1.0,1.0,0.016266316846322296,1.0,1.0,,,,,, +146,6994,146,0.023999675555041906,1.0,1.0,,,,,,,,, +147,7035,147,0.01443589308463561,1.0,1.0,,,,,,,,, +148,7124,148,0.01593858884255083,1.0,1.0,,,,,,,,, +149,7157,149,0.0039451327360802676,1.0,1.0,,,,,,,,, +150,7286,150,0.041224458131234765,1.0,1.0,,,,,,,,, +151,7327,151,0.008721806495872442,1.0,1.0,,,,,,,,, +152,7360,152,0.003449108358909143,1.0,1.0,,,,,,,,, +153,7441,153,0.031120321435428196,1.0,1.0,0.011828955701554756,1.0,1.0,,,,,, +154,7482,154,0.009850116068482748,1.0,1.0,,,,,,,,, +155,7515,155,0.01212568152328231,1.0,1.0,,,,,,,,, +156,7540,156,0.009057329903413725,1.0,1.0,,,,,,,,, +157,7573,157,0.008076899820198376,1.0,1.0,,,,,,,,, +158,7598,158,0.00785573707279582,1.0,1.0,,,,,,,,, +159,7623,159,0.0109080523573175,1.0,1.0,,,,,,,,, +160,7680,160,0.01987976644248452,1.0,1.0,,,,,,,,, +161,7705,161,0.010745363941047492,1.0,1.0,0.010937099401571686,1.0,1.0,,,,,, +162,7730,162,0.30798120539652396,0.0,0.0,,,,,,,,, +163,7763,163,0.00899759478130818,1.0,1.0,,,,,,,,, +164,7804,164,0.01044557543134579,1.0,1.0,,,,,,,,, +165,7837,165,0.03216896385778455,1.0,1.0,,,,,,,,, +166,7902,166,0.025793771446051328,1.0,1.0,,,,,,,,, +167,7927,167,0.016294540036627343,1.0,1.0,,,,,,,,, +168,7952,168,0.01383792414747101,1.0,1.0,,,,,,,,, +169,8073,169,0.0442201690788764,1.0,1.0,0.019650482941906376,1.0,1.0,,,,,, +170,8114,170,0.010494683315548172,1.0,1.0,,,,,,,,, +171,8155,171,0.021872556327361962,1.0,1.0,,,,,,,,, +172,8188,172,0.010315648031782984,1.0,1.0,,,,,,,,, +173,8253,173,0.09507020872195425,0.0,0.0,,,,,,,,, +174,8350,174,0.03518366060497404,1.0,1.0,,,,,,,,, +175,8391,175,0.02231018475806923,1.0,1.0,,,,,,,,, +176,8424,176,0.07556836719186322,0.0,0.0,,,,,,,,, +177,8465,177,0.01612279840865813,1.0,1.0,0.009804825048073933,1.0,1.0,,,,,, +178,8506,178,0.015526040938086228,1.0,1.0,,,,,,,,, +179,8531,179,0.007676232928230699,1.0,1.0,,,,,,,,, +180,8572,180,0.015856357650445716,1.0,1.0,,,,,,,,, +181,8605,181,0.04266274086521662,0.0,0.0,,,,,,,,, +182,8630,182,0.017102236310286872,1.0,1.0,,,,,,,,, +183,8655,183,0.023506063156340434,1.0,1.0,,,,,,,,, +184,8688,184,0.010906938337102476,1.0,1.0,,,,,,,,, +185,8753,185,0.02067501008144973,1.0,1.0,0.01946405619877017,1.0,1.0,0.9347826242446899,0.9696969696969697,0.9411764705882353,0.9950980392156863,0.9982683982683983,0.955223880597015 +186,8818,186,0.0253877446284514,1.0,1.0,,,,,,,,, +187,8843,187,0.008552278111844011,1.0,1.0,,,,,,,,, +188,8876,188,0.004885866508084292,1.0,1.0,,,,,,,,, +189,8909,189,0.0096407152939355,1.0,1.0,,,,,,,,, +190,9038,190,0.041447347311209345,1.0,1.0,,,,,,,,, +191,9079,191,0.008069531135470242,1.0,1.0,,,,,,,,, +192,9112,192,0.019019369105776604,1.0,1.0,,,,,,,,, +193,9233,193,0.04434483481046724,1.0,1.0,0.0045015245680891285,1.0,1.0,,,,,, +194,9274,194,0.007292373489035527,1.0,1.0,,,,,,,,, +195,9299,195,0.0043528404197422784,1.0,1.0,,,,,,,,, +196,9340,196,0.008813853941541312,1.0,1.0,,,,,,,,, +197,9365,197,0.006371431822229472,1.0,1.0,,,,,,,,, +198,9406,198,0.00623902668224946,1.0,1.0,,,,,,,,, +199,9439,199,0.012713423611144208,1.0,1.0,,,,,,,,, +200,9480,200,0.01829142275457343,1.0,1.0,,,,,,,,, +201,9705,201,0.028767533754505945,1.0,1.0,0.003386026440469625,1.0,1.0,,,,,, +202,9738,202,0.03952259193970577,0.0,0.0,,,,,,,,, +203,9851,203,0.045343604889194145,1.0,1.0,,,,,,,,, +204,9892,204,0.008729404071214503,1.0,1.0,,,,,,,,, +205,10013,205,0.04341929623221237,1.0,1.0,,,,,,,,, +206,10054,206,0.007295288263360816,1.0,1.0,,,,,,,,, +207,10095,207,0.008120490976995824,1.0,1.0,,,,,,,,, +208,10136,208,0.01641764993843071,1.0,1.0,,,,,,,,, +209,10161,209,0.006405777646996285,1.0,1.0,0.01597132712557709,1.0,1.0,,,,,, +210,10194,210,0.0025642871842273965,1.0,1.0,,,,,,,,, +211,10235,211,0.004703610731382924,1.0,1.0,,,,,,,,, +212,10268,212,0.02592917215889884,1.0,1.0,,,,,,,,, +213,10309,213,0.008380668578890435,1.0,1.0,,,,,,,,, +214,10350,214,0.008051953344805952,1.0,1.0,,,,,,,,, +215,10375,215,0.007385871251860104,1.0,1.0,,,,,,,,, +216,10416,216,0.005717272297546515,1.0,1.0,,,,,,,,, +217,10449,217,0.015528442430270389,1.0,1.0,0.01990651841883543,1.0,1.0,,,,,, +218,10482,218,0.02768984882632594,0.0,0.0,,,,,,,,, +219,10515,219,0.01808141203696323,1.0,1.0,,,,,,,,, +220,10548,220,0.0014204642545382741,1.0,1.0,,,,,,,,, +221,10581,221,0.014154038157057586,1.0,1.0,,,,,,,,, +222,10622,222,0.007296723900188227,1.0,1.0,,,,,,,,, +223,10655,223,0.00897067299892342,1.0,1.0,,,,,,,,, +224,10680,224,0.005515119432669356,1.0,1.0,,,,,,,,, +225,10713,225,0.007543390910368543,1.0,1.0,0.007103388840405386,1.0,1.0,,,,,, +226,10754,226,0.007296255969748019,1.0,1.0,,,,,,,,, +227,10779,227,0.00534076511762881,1.0,1.0,,,,,,,,, +228,10868,228,0.01642789892993113,1.0,1.0,,,,,,,,, +229,10909,229,0.009295455787083485,1.0,1.0,,,,,,,,, +230,10934,230,0.003962355921809527,1.0,1.0,,,,,,,,, +231,10959,231,0.007570776468971113,1.0,1.0,,,,,,,,, +232,10992,232,0.002741338354686604,1.0,1.0,,,,,,,,, +233,11025,233,0.007267029586116678,1.0,1.0,0.011444191882051197,1.0,1.0,,,,,, +234,11066,234,0.004381624559087869,1.0,1.0,,,,,,,,, +235,11091,235,0.005789060645329301,1.0,1.0,,,,,,,,, +236,11124,236,0.004000034101801475,1.0,1.0,,,,,,,,, +237,11189,237,0.018686159788669554,1.0,1.0,,,,,,,,, +238,11214,238,0.007860034823861497,1.0,1.0,,,,,,,,, +239,11239,239,0.0026639150511550726,1.0,1.0,,,,,,,,, +240,11352,240,0.04654545919382193,1.0,1.0,,,,,,,,, +241,11377,241,0.002984911914496706,1.0,1.0,0.01610577152310356,1.0,1.0,,,,,, +242,11410,242,0.006983030153049299,1.0,1.0,,,,,,,,, +243,11435,243,0.2008603759948039,0.0,0.0,,,,,,,,, +244,11476,244,0.009287042262392789,1.0,1.0,,,,,,,,, +245,11525,245,0.019457386839599924,1.0,1.0,,,,,,,,, +246,11574,246,0.018471286773068793,1.0,1.0,,,,,,,,, +247,11615,247,0.004312932416312833,1.0,1.0,,,,,,,,, +248,11656,248,0.0036243171682654065,1.0,1.0,,,,,,,,, +249,11745,249,0.015389378262052975,1.0,1.0,0.013050893215513499,1.0,1.0,,,,,, +250,11818,250,0.030484877124771527,1.0,1.0,,,,,,,,, +251,11851,251,0.00902285473648135,1.0,1.0,,,,,,,,, +252,11884,252,0.008806582555814658,1.0,1.0,,,,,,,,, +253,12013,253,0.04047198609091396,1.0,1.0,,,,,,,,, +254,12054,254,0.0050232971199402525,1.0,1.0,,,,,,,,, +255,12159,255,0.03146750575378414,1.0,1.0,,,,,,,,, +256,12184,256,0.00786894164726012,1.0,1.0,,,,,,,,, +257,12209,257,0.00972143760251735,1.0,1.0,0.014077054748410471,1.0,1.0,,,,,, +258,12282,258,0.03361915087383548,1.0,1.0,,,,,,,,, +259,12307,259,0.052315172771777725,0.0,0.0,,,,,,,,, +260,12348,260,0.01053579846235516,1.0,1.0,,,,,,,,, +261,12381,261,0.0031658054974104746,1.0,1.0,,,,,,,,, +262,12422,262,0.01205441480271111,1.0,1.0,,,,,,,,, +263,12463,263,0.010403739995588697,1.0,1.0,,,,,,,,, +264,12488,264,0.0070665811240238924,1.0,1.0,,,,,,,,, +265,12529,265,0.00961365086103615,1.0,1.0,0.00905852913311719,1.0,1.0,,,,,, +266,12562,266,0.013667365071232779,1.0,1.0,,,,,,,,, +267,12595,267,0.023633310047362105,1.0,1.0,,,,,,,,, +268,12708,268,0.04543655335229817,1.0,1.0,,,,,,,,, +269,12741,269,0.010956583107381448,1.0,1.0,,,,,,,,, +270,12774,270,0.014410940355542154,1.0,1.0,,,,,,,,, +271,12887,271,0.04542291178739106,1.0,1.0,,,,,,,,, +272,12912,272,0.00994476468948657,1.0,1.0,,,,,,,,, +273,12937,273,0.008386650285174255,1.0,1.0,0.014342974209212114,1.0,1.0,,,,,, +274,12994,274,0.018147266913760434,1.0,1.0,,,,,,,,, +275,13035,275,0.006538654259070429,1.0,1.0,,,,,,,,, +276,13068,276,0.009877847383381776,1.0,1.0,,,,,,,,, +277,13101,277,0.0031727705229150165,1.0,1.0,,,,,,,,, +278,13166,278,0.016307113373967466,1.0,1.0,,,,,,,,, +279,13191,279,0.005887442197757385,1.0,1.0,,,,,,,,, +280,13280,280,0.01564549290126293,1.0,1.0,,,,,,,,, +281,13305,281,0.005752370594069968,1.0,1.0,0.006978824477537274,1.0,1.0,,,,,, +282,13346,282,0.016335168046240144,1.0,1.0,,,,,,,,, +283,13371,283,0.0650733174790609,0.0,0.0,,,,,,,,, +284,13396,284,0.002810546016518553,1.0,1.0,,,,,,,,, +285,13485,285,0.014537198734476328,1.0,1.0,,,,,,,,, +286,13566,286,0.022614223145050575,1.0,1.0,,,,,,,,, +287,13599,287,0.004838370113207021,1.0,1.0,,,,,,,,, +288,13632,288,0.04014172601554037,0.0,0.0,,,,,,,,, +289,13657,289,0.0068272184850532835,1.0,1.0,0.002899069070954283,1.0,1.0,,,,,, +290,13778,290,0.04056780969254245,1.0,1.0,,,,,,,,, +291,13803,291,0.0028586753122666173,1.0,1.0,,,,,,,,, +292,13868,292,0.19208342062729516,0.0,0.0,,,,,,,,, +293,13893,293,0.010763978790896545,1.0,1.0,,,,,,,,, +294,13934,294,0.0067817636981822044,1.0,1.0,,,,,,,,, +295,13967,295,0.009486361342342207,1.0,1.0,,,,,,,,, +296,14000,296,0.1134855329726662,0.0,0.0,,,,,,,,, +297,14065,297,0.014872436244601772,1.0,1.0,0.003247401493712611,1.0,1.0,,,,,, +298,14090,298,0.008453926157593135,1.0,1.0,,,,,,,,, +299,14115,299,0.006478150784141689,1.0,1.0,,,,,,,,, +300,14140,300,0.006300340815787422,1.0,1.0,,,,,,,,, +301,14205,301,0.015484150805193622,1.0,1.0,,,,,,,,, +302,14230,302,0.004749118860492368,1.0,1.0,,,,,,,,, +303,14319,303,0.018857628523574022,1.0,1.0,,,,,,,,, +304,14360,304,0.009748939698543837,1.0,1.0,,,,,,,,, +305,14401,305,0.009366595859616609,1.0,1.0,0.0035716364178006183,1.0,1.0,,,,,, +306,14498,306,0.03193726985777782,1.0,1.0,,,,,,,,, +307,14619,307,0.04245903858144251,1.0,1.0,,,,,,,,, +308,14644,308,0.007661057935747522,1.0,1.0,,,,,,,,, +309,14685,309,0.0070795894133632385,1.0,1.0,,,,,,,,, +310,14726,310,0.007909869037675888,1.0,1.0,,,,,,,,, +311,14767,311,0.008039157802236333,1.0,1.0,,,,,,,,, +312,14792,312,0.004267013477024193,1.0,1.0,,,,,,,,, +313,14817,313,0.0069021424798408285,1.0,1.0,0.012983708210138046,1.0,1.0,,,,,, +314,14858,314,0.005970162558935954,1.0,1.0,,,,,,,,, +315,14899,315,0.005226014579985986,1.0,1.0,,,,,,,,, +316,14924,316,0.004436972953249096,1.0,1.0,,,,,,,,, +317,15005,317,0.02090917988610454,1.0,1.0,,,,,,,,, +318,15038,318,0.01687274852518158,1.0,1.0,,,,,,,,, +319,15063,319,0.0034286084494730557,1.0,1.0,,,,,,,,, +320,15104,320,0.0058259491427456855,1.0,1.0,,,,,,,,, +321,15129,321,0.014218641348834354,1.0,1.0,0.009462272379369096,1.0,1.0,,,,,, +322,15250,322,0.041101253618656915,1.0,1.0,,,,,,,,, +323,15283,323,0.007723327267934224,1.0,1.0,,,,,,,,, +324,15372,324,0.01400106868225769,1.0,1.0,,,,,,,,, +325,15413,325,0.0056402808669060965,1.0,1.0,,,,,,,,, +326,15502,326,0.01330174182049053,1.0,1.0,,,,,,,,, +327,15535,327,0.012951216341078292,1.0,1.0,,,,,,,,, +328,15568,328,0.007702305408448921,1.0,1.0,,,,,,,,, +329,15593,329,0.00233715995310528,1.0,1.0,0.10090430374459175,0.0,0.0,,,,,, +330,15618,330,0.004710533475930181,1.0,1.0,,,,,,,,, +331,15651,331,0.007741873095687879,1.0,1.0,,,,,,,,, +332,15716,332,0.010987720771966563,1.0,1.0,,,,,,,,, +333,15741,333,0.005325472617975416,1.0,1.0,,,,,,,,, +334,15766,334,0.007381831800666987,1.0,1.0,,,,,,,,, +335,15791,335,0.019197925892998076,1.0,1.0,,,,,,,,, +336,15824,336,0.01228676644600455,1.0,1.0,,,,,,,,, +337,15865,337,0.007088923838839379,1.0,1.0,0.0032772433804825595,1.0,1.0,,,,,, +338,15906,338,0.007022953629755194,1.0,1.0,,,,,,,,, +339,15931,339,0.005435457404794307,1.0,1.0,,,,,,,,, +340,16052,340,0.04093561813934321,1.0,1.0,,,,,,,,, +341,16077,341,0.009327377094967535,1.0,1.0,,,,,,,,, +342,16118,342,0.006350644650986397,1.0,1.0,,,,,,,,, +343,16151,343,0.011814881276115444,1.0,1.0,,,,,,,,, +344,16192,344,0.004296123096157316,1.0,1.0,,,,,,,,, +345,16233,345,0.005230187705348726,1.0,1.0,0.03927984390400979,0.0,0.0,,,,,, +346,16266,346,0.010122379710619496,1.0,1.0,,,,,,,,, +347,16387,347,0.0406074109057178,1.0,1.0,,,,,,,,, +348,16420,348,0.011784737493061357,1.0,1.0,,,,,,,,, +349,16533,349,0.04488378147586867,1.0,1.0,,,,,,,,, +350,16574,350,0.006899318956178832,1.0,1.0,,,,,,,,, +351,16615,351,0.00632328103694772,1.0,1.0,,,,,,,,, +352,16640,352,0.006382795259487466,1.0,1.0,,,,,,,,, +353,16673,353,0.0068136124559623566,1.0,1.0,0.01830251012032386,1.0,1.0,,,,,, +354,16706,354,0.004431937555007153,1.0,1.0,,,,,,,,, +355,16747,355,0.006779200907094492,1.0,1.0,,,,,,,,, +356,16788,356,0.005477589684106989,1.0,1.0,,,,,,,,, +357,16821,357,0.0027187009936051254,1.0,1.0,,,,,,,,, +358,16854,358,0.00746680791613926,1.0,1.0,,,,,,,,, +359,16895,359,0.004985898222737888,1.0,1.0,,,,,,,,, +360,16928,360,0.007016382109242727,1.0,1.0,,,,,,,,, +361,17001,361,0.06241597279883636,0.0,0.0,0.025818622997012423,1.0,1.0,,,,,, +362,17090,362,0.012982501689402232,1.0,1.0,,,,,,,,, +363,17171,363,0.01858814670528591,1.0,1.0,,,,,,,,, +364,17236,364,0.019674304315917653,1.0,1.0,,,,,,,,, +365,17277,365,0.0065508874998195105,1.0,1.0,,,,,,,,, +366,17310,366,0.0060593176436691465,1.0,1.0,,,,,,,,, +367,17343,367,0.007509909824569263,1.0,1.0,,,,,,,,, +368,17376,368,0.00289684487633167,1.0,1.0,,,,,,,,, +369,17409,369,0.009802874652941683,1.0,1.0,0.009429889484254092,1.0,1.0,,,,,, +370,17450,370,0.006170175576537078,1.0,1.0,,,,0.97826087474823,0.9714285714285714,1.0,1.0,1.0,0.9855072463768115 +371,17475,371,0.004977268490226076,1.0,1.0,,,,,,,,, +372,17508,372,0.00848211291605253,1.0,1.0,,,,,,,,, +373,17533,373,0.009588292055759666,1.0,1.0,,,,,,,,, +374,17574,374,0.0052706008605209725,1.0,1.0,,,,,,,,, +375,17599,375,0.005479556109006201,1.0,1.0,,,,,,,,, +376,17624,376,0.0051099724760539774,1.0,1.0,,,,,,,,, +377,17657,377,0.008797225462471941,1.0,1.0,0.010136134883024965,1.0,1.0,,,,,, +378,17786,378,0.037386029059273934,1.0,1.0,,,,,,,,, +379,17827,379,0.005779493440183615,1.0,1.0,,,,,,,,, +380,17868,380,0.005227617102249043,1.0,1.0,,,,,,,,, +381,17901,381,0.00872914082882403,1.0,1.0,,,,,,,,, +382,17982,382,0.017498087919149842,1.0,1.0,,,,,,,,, +383,18015,383,0.00790408858559431,1.0,1.0,,,,,,,,, +384,18056,384,0.005077853533686722,1.0,1.0,,,,,,,,, +385,18089,385,0.0036631167140444634,1.0,1.0,0.012814363635228193,1.0,1.0,,,,,, +386,18122,386,0.0101900941110614,1.0,1.0,,,,,,,,, +387,18147,387,0.008469455682691205,1.0,1.0,,,,,,,,, +388,18188,388,0.00446530517785368,1.0,1.0,,,,,,,,, +389,18213,389,0.0030794867682288896,1.0,1.0,,,,,,,,, +390,18238,390,0.004241252369480813,1.0,1.0,,,,,,,,, +391,18327,391,0.012792469690022533,1.0,1.0,,,,,,,,, +392,18368,392,0.006121942382329962,1.0,1.0,,,,,,,,, +393,18433,393,0.019442220682030893,1.0,1.0,0.006381066696098401,1.0,1.0,,,,,, +394,18474,394,0.004459289899687747,1.0,1.0,,,,,,,,, +395,18515,395,0.006409174696565287,1.0,1.0,,,,,,,,, +396,18540,396,0.0029228208776875893,1.0,1.0,,,,,,,,, +397,18645,397,0.026463443300660333,1.0,1.0,,,,,,,,, +398,18670,398,0.005060745925981488,1.0,1.0,,,,,,,,, +399,18695,399,0.00277868624844397,1.0,1.0,,,,,,,,, +400,18728,400,0.005215658468948732,1.0,1.0,,,,,,,,, +401,18753,401,0.014685201665655161,1.0,1.0,0.0025610368884247236,1.0,1.0,,,,,, +402,18826,402,0.03270863481563032,1.0,1.0,,,,,,,,, +403,18947,403,0.039293468341419155,1.0,1.0,,,,,,,,, +404,18972,404,0.005347372380997512,1.0,1.0,,,,,,,,, +405,19069,405,0.029180476018126834,1.0,1.0,,,,,,,,, +406,19182,406,0.04261210803226927,1.0,1.0,,,,,,,,, +407,19215,407,0.01285147528273882,1.0,1.0,,,,,,,,, +408,19256,408,0.0051810646574761005,1.0,1.0,,,,,,,,, +409,19297,409,0.004588275055159101,1.0,1.0,0.010190846248395929,1.0,1.0,,,,,, +410,19330,410,0.005118333911291302,1.0,1.0,,,,,,,,, +411,19363,411,0.007199785555521317,1.0,1.0,,,,,,,,, +412,19396,412,0.007523534953842841,1.0,1.0,,,,,,,,, +413,19437,413,0.0036133376600282646,1.0,1.0,,,,,,,,, +414,19470,414,0.00754366617161395,1.0,1.0,,,,,,,,, +415,19519,415,0.013063631419148198,1.0,1.0,,,,,,,,, +416,19560,416,0.006516043173423214,1.0,1.0,,,,,,,,, +417,19585,417,0.005909063258161117,1.0,1.0,0.004484898516798239,1.0,1.0,,,,,, +418,19610,418,0.002752600935330823,1.0,1.0,,,,,,,,, +419,19651,419,0.013112167946243973,1.0,1.0,,,,,,,,, +420,19740,420,0.010890522728516937,1.0,1.0,,,,,,,,, +421,19781,421,0.0034274939357728735,1.0,1.0,,,,,,,,, +422,19822,422,0.004551760228792545,1.0,1.0,,,,,,,,, +423,19863,423,0.005666332393690141,1.0,1.0,,,,,,,,, +424,19952,424,0.010089414473750159,1.0,1.0,,,,,,,,, +425,19977,425,0.005421592725300949,1.0,1.0,0.010154089158733087,1.0,1.0,,,,,, +426,20010,426,0.013995204517200365,1.0,1.0,,,,,,,,, +427,20043,427,0.001774006145708995,1.0,1.0,,,,,,,,, +428,20156,428,0.04210431421705575,1.0,1.0,,,,,,,,, +429,20197,429,0.008133097550726234,1.0,1.0,,,,,,,,, +430,20230,430,0.005435841163606198,1.0,1.0,,,,,,,,, +431,20271,431,0.003560666549067719,1.0,1.0,,,,,,,,, +432,20304,432,0.00933159835393269,1.0,1.0,,,,,,,,, +433,20337,433,0.0039049111738194138,1.0,1.0,0.0028419618421814145,1.0,1.0,,,,,, +434,20362,434,0.002813171208986963,1.0,1.0,,,,,,,,, +435,20427,435,0.0682248028461538,0.0,0.0,,,,,,,,, +436,20468,436,0.00482745176948265,1.0,1.0,,,,,,,,, +437,20589,437,0.0399061410026543,1.0,1.0,,,,,,,,, +438,20622,438,0.003047735501219791,1.0,1.0,,,,,,,,, +439,20687,439,0.012574506491208694,1.0,1.0,,,,,,,,, +440,20720,440,0.004962131093689095,1.0,1.0,,,,,,,,, +441,20761,441,0.005481797426770754,1.0,1.0,0.01409145276248637,1.0,1.0,,,,,, +442,20882,442,0.03914496499334654,1.0,1.0,,,,,,,,, +443,20923,443,0.00289314573109823,1.0,1.0,,,,,,,,, +444,21044,444,0.04075343354686266,1.0,1.0,,,,,,,,, +445,21069,445,0.0024204443084120918,1.0,1.0,,,,,,,,, +446,21094,446,0.0029809100659281282,1.0,1.0,,,,,,,,, +447,21183,447,0.010707999333096517,1.0,1.0,,,,,,,,, +448,21208,448,0.010865561153541422,1.0,1.0,,,,,,,,, +449,21233,449,0.0043157189534379845,1.0,1.0,0.01338142135711223,1.0,1.0,,,,,, +450,21290,450,0.040722006902027964,0.0,0.0,,,,,,,,, +451,21323,451,0.008621539468936136,1.0,1.0,,,,,,,,, +452,21356,452,0.0016761923395642953,1.0,1.0,,,,,,,,, +453,21381,453,0.005994952324253973,1.0,1.0,,,,,,,,, +454,21406,454,0.0037538246275077352,1.0,1.0,,,,,,,,, +455,21471,455,0.0195343103815992,1.0,1.0,,,,,,,,, +456,21552,456,0.015445701708838896,1.0,1.0,,,,,,,,, +457,21593,457,0.00434749203245974,1.0,1.0,0.00894231928634099,1.0,1.0,,,,,, +458,21626,458,0.0034527844001934226,1.0,1.0,,,,,,,,, +459,21659,459,0.0031894931510612424,1.0,1.0,,,,,,,,, +460,21700,460,0.006365930143310493,1.0,1.0,,,,,,,,, +461,21733,461,0.008079833049435336,1.0,1.0,,,,,,,,, +462,21846,462,0.04329203566561464,1.0,1.0,,,,,,,,, +463,21967,463,0.039069228057969356,1.0,1.0,,,,,,,,, +464,22000,464,0.008768390877131651,1.0,1.0,,,,,,,,, +465,22041,465,0.014477506232410634,1.0,1.0,0.003773284602657441,1.0,1.0,,,,,, +466,22074,466,0.006454803282881017,1.0,1.0,,,,,,,,, +467,22115,467,0.006094551668831627,1.0,1.0,,,,,,,,, +468,22236,468,0.03899099508015843,1.0,1.0,,,,,,,,, +469,22277,469,0.0044074383070502,1.0,1.0,,,,,,,,, +470,22310,470,0.003730343071833886,1.0,1.0,,,,,,,,, +471,22335,471,0.0019561042940116143,1.0,1.0,,,,,,,,, +472,22464,472,0.03706168061357318,1.0,1.0,,,,,,,,, +473,22513,473,0.009730118836945013,1.0,1.0,0.004400895589583342,1.0,1.0,,,,,, +474,22538,474,0.005502700208481884,1.0,1.0,,,,,,,,, +475,22563,475,0.003366184840212251,1.0,1.0,,,,,,,,, +476,22596,476,0.004876155591334737,1.0,1.0,,,,,,,,, +477,22685,477,0.007369980880529594,1.0,1.0,,,,,,,,, +478,22726,478,0.00423608940248505,1.0,1.0,,,,,,,,, +479,22759,479,0.005535398244869327,1.0,1.0,,,,,,,,, +480,22800,480,0.004159781519101672,1.0,1.0,,,,,,,,, +481,22833,481,0.002388506317552128,1.0,1.0,0.0029051203060234346,1.0,1.0,,,,,, +482,22874,482,0.004042849869679457,1.0,1.0,,,,,,,,, +483,22907,483,0.0039954076296243105,1.0,1.0,,,,,,,,, +484,22940,484,0.003777253892267254,1.0,1.0,,,,,,,,, +485,23029,485,0.005931295647388157,1.0,1.0,,,,,,,,, +486,23142,486,0.041633564051607466,1.0,1.0,,,,,,,,, +487,23175,487,0.002843823245318617,1.0,1.0,,,,,,,,, +488,23216,488,0.004904347977841099,1.0,1.0,,,,,,,,, +489,23281,489,0.007685642883870116,1.0,1.0,0.010009516712578563,1.0,1.0,,,,,, +490,23370,490,0.004961004897752446,1.0,1.0,,,,,,,,, +491,23403,491,0.14175609523382882,0.0,0.0,,,,,,,,, +492,23444,492,0.004405457678391486,1.0,1.0,,,,,,,,, +493,23469,493,0.0029353828574415763,1.0,1.0,,,,,,,,, +494,23502,494,0.008325506376517779,1.0,1.0,,,,,,,,, +495,23727,495,0.030515501271879658,1.0,1.0,,,,,,,,, +496,23768,496,0.003250050957994745,1.0,1.0,,,,,,,,, +497,23793,497,0.0024827693286867242,1.0,1.0,0.003103244707554704,1.0,1.0,,,,,, +498,23834,498,0.003702679923829848,1.0,1.0,,,,,,,,, +499,23867,499,0.005541484779618031,1.0,1.0,,,,,,,,, +500,23900,500,0.005214432409935057,1.0,1.0,,,,,,,,, +501,23925,501,0.0019390572062438116,1.0,1.0,,,,,,,,, +502,23990,502,0.006657855856123263,1.0,1.0,,,,,,,,, +503,24031,503,0.005829024780737201,1.0,1.0,,,,,,,,, +504,24056,504,0.006175469648775713,1.0,1.0,,,,,,,,, +505,24097,505,0.0038455014436263473,1.0,1.0,0.0036590021965975696,1.0,1.0,,,,,, +506,24138,506,0.006219480526445978,1.0,1.0,,,,,,,,, +507,24179,507,0.006098381495399053,1.0,1.0,,,,,,,,, +508,24204,508,0.007283948691612066,1.0,1.0,,,,,,,,, +509,24229,509,0.002395971716434815,1.0,1.0,,,,,,,,, +510,24302,510,0.029476810715614192,1.0,1.0,,,,,,,,, +511,24375,511,0.03041542010091386,1.0,1.0,,,,,,,,, +512,24416,512,0.005799273666939143,1.0,1.0,,,,,,,,, +513,24441,513,0.005022879587259012,1.0,1.0,0.006547132338243102,1.0,1.0,,,,,, +514,24482,514,0.0163119795171624,1.0,1.0,,,,,,,,, +515,24523,515,0.005791362401192259,1.0,1.0,,,,,,,,, +516,24644,516,0.038940206215442544,1.0,1.0,,,,,,,,, +517,24709,517,0.01761021553312844,1.0,1.0,,,,,,,,, +518,24742,518,0.0013513698613417094,1.0,1.0,,,,,,,,, +519,24783,519,0.0029700993190883767,1.0,1.0,,,,,,,,, +520,24808,520,0.003247842869454871,1.0,1.0,,,,,,,,, +521,24849,521,0.0045765779717933126,1.0,1.0,0.012771317202601723,1.0,1.0,,,,,, +522,24938,522,0.006073696234293493,1.0,1.0,,,,,,,,, +523,24979,523,0.005132250493824587,1.0,1.0,,,,,,,,, +524,25020,524,0.003921822247708833,1.0,1.0,,,,,,,,, +525,25061,525,0.0035924963277687142,1.0,1.0,,,,,,,,, +526,25086,526,0.00283525526045499,1.0,1.0,,,,,,,,, +527,25127,527,0.0030496149844513815,1.0,1.0,,,,,,,,, +528,25160,528,0.002263123477607678,1.0,1.0,,,,,,,,, +529,25185,529,0.0045095226738976635,1.0,1.0,0.0018520656340744303,1.0,1.0,,,,,, +530,25218,530,0.0058128219632149584,1.0,1.0,,,,,,,,, +531,25243,531,0.00197971886399273,1.0,1.0,,,,,,,,, +532,25268,532,0.002230205244842788,1.0,1.0,,,,,,,,, +533,25293,533,0.007282811450522786,1.0,1.0,,,,,,,,, +534,25358,534,0.0054789967322227545,1.0,1.0,,,,,,,,, +535,25439,535,0.021891684533620268,1.0,1.0,,,,,,,,, +536,25504,536,0.005303770605250975,1.0,1.0,,,,,,,,, +537,25529,537,0.002014399528134163,1.0,1.0,0.0042794912586691675,1.0,1.0,,,,,, +538,25562,538,0.002943645314093857,1.0,1.0,,,,,,,,, +539,25587,539,0.0018909434367689452,1.0,1.0,,,,,,,,, +540,25612,540,0.0021647153882243704,1.0,1.0,,,,,,,,, +541,25645,541,0.006050590580967139,1.0,1.0,,,,,,,,, +542,25678,542,0.0033984149457842255,1.0,1.0,,,,,,,,, +543,25703,543,0.004811875453335688,1.0,1.0,,,,,,,,, +544,25728,544,0.004672682977889811,1.0,1.0,,,,,,,,, +545,25753,545,0.040968753699232814,0.0,0.0,0.004558271169915428,1.0,1.0,,,,,, +546,25866,546,0.04114297590619628,1.0,1.0,,,,,,,,, +547,25907,547,0.0026330222110076567,1.0,1.0,,,,,,,,, +548,25932,548,0.008056314813032942,1.0,1.0,,,,,,,,, +549,25965,549,0.005396297170821356,1.0,1.0,,,,,,,,, +550,26054,550,0.004538955895886829,1.0,1.0,,,,,,,,, +551,26119,551,0.01757690426377849,1.0,1.0,,,,0.97826087474823,0.9714285714285714,1.0,0.9975490196078431,0.999147305981216,0.9855072463768115 +552,26160,552,0.013831619186128595,1.0,1.0,,,,,,,,, +553,26201,553,0.0034254586723514537,1.0,1.0,0.008426652172658273,1.0,1.0,,,,,, +554,26226,554,0.00341066106762355,1.0,1.0,,,,,,,,, +555,26259,555,0.008006247479813738,1.0,1.0,,,,,,,,, +556,26300,556,0.005676790891738619,1.0,1.0,,,,,,,,, +557,26341,557,0.005004796889145837,1.0,1.0,,,,,,,,, +558,26366,558,0.010027413435147233,1.0,1.0,,,,,,,,, +559,26399,559,0.004296689486836556,1.0,1.0,,,,,,,,, +560,26440,560,0.004505495261534521,1.0,1.0,,,,,,,,, +561,26473,561,0.006024071594807481,1.0,1.0,0.00201452387670732,1.0,1.0,,,,,, +562,26498,562,0.0033772755150941218,1.0,1.0,,,,,,,,, +563,26531,563,0.0012674026753745694,1.0,1.0,,,,,,,,, +564,26596,564,0.0037556562418396312,1.0,1.0,,,,,,,,, +565,26621,565,0.005599753787028779,1.0,1.0,,,,,,,,, +566,26654,566,0.002312305997482511,1.0,1.0,,,,,,,,, +567,26687,567,0.0073554601565973595,1.0,1.0,,,,,,,,, +568,26728,568,0.0038110892817176142,1.0,1.0,,,,,,,,, +569,26753,569,0.002603204145544661,1.0,1.0,0.003431684652123437,1.0,1.0,,,,,, +570,26786,570,0.002286822066678269,1.0,1.0,,,,,,,,, +571,26851,571,0.003398716106238738,1.0,1.0,,,,,,,,, +572,26916,572,0.01528584834675371,1.0,1.0,,,,,,,,, +573,26957,573,0.0026017046176115316,1.0,1.0,,,,,,,,, +574,26998,574,0.003980031256530766,1.0,1.0,,,,,,,,, +575,27063,575,0.003080047596640441,1.0,1.0,,,,,,,,, +576,27096,576,0.0014191712232921817,1.0,1.0,,,,,,,,, +577,27137,577,0.0027430418360495424,1.0,1.0,0.07943665660214899,0.0,0.0,,,,,, +578,27202,578,0.01588205919491429,1.0,1.0,,,,,,,,, +579,27235,579,0.006097173654818561,1.0,1.0,,,,,,,,, +580,27316,580,0.013398075245790095,1.0,1.0,,,,,,,,, +581,27349,581,0.006501001887005202,1.0,1.0,,,,,,,,, +582,27382,582,0.006148814943287633,1.0,1.0,,,,,,,,, +583,27423,583,0.0022625561114754194,1.0,1.0,,,,,,,,, +584,27520,584,0.02587177884219007,1.0,1.0,,,,,,,,, +585,27553,585,0.00427734150788595,1.0,1.0,0.01190931943495305,1.0,1.0,,,,,, +586,27642,586,0.003979477948081777,1.0,1.0,,,,,,,,, +587,27675,587,0.00523442531635311,1.0,1.0,,,,,,,,, +588,27700,588,0.004187755111752267,1.0,1.0,,,,,,,,, +589,27749,589,0.007824519231317783,1.0,1.0,,,,,,,,, +590,27774,590,0.002154234060495052,1.0,1.0,,,,,,,,, +591,27815,591,0.004621449108847628,1.0,1.0,,,,,,,,, +592,27848,592,0.0014139314450412144,1.0,1.0,,,,,,,,, +593,27889,593,0.0022663614025283324,1.0,1.0,0.0070322001175645715,1.0,1.0,,,,,, +594,27922,594,0.0058935468012361976,1.0,1.0,,,,,,,,, +595,27955,595,0.003369732270692509,1.0,1.0,,,,,,,,, +596,27980,596,0.003596647399619997,1.0,1.0,,,,,,,,, +597,28005,597,0.002301241029939926,1.0,1.0,,,,,,,,, +598,28054,598,0.006531533787848881,1.0,1.0,,,,,,,,, +599,28095,599,0.003337418648904532,1.0,1.0,,,,,,,,, +600,28128,600,0.0035513282102157006,1.0,1.0,,,,,,,,, +601,28169,601,0.003965859641060888,1.0,1.0,0.0029647432636147643,1.0,1.0,,,,,, +602,28194,602,0.0019383963347278648,1.0,1.0,,,,,,,,, +603,28283,603,0.003140089869592141,1.0,1.0,,,,,,,,, +604,28324,604,0.00486154903698423,1.0,1.0,,,,,,,,, +605,28365,605,0.0047934978819980075,1.0,1.0,,,,,,,,, +606,28390,606,0.0017501254036809161,1.0,1.0,,,,,,,,, +607,28463,607,0.026914087987066876,1.0,1.0,,,,,,,,, +608,28488,608,0.006668408524456785,1.0,1.0,,,,,,,,, +609,28513,609,0.0021966585724375893,1.0,1.0,0.013416350759023317,1.0,1.0,,,,,, +610,28634,610,0.038277191446451905,1.0,1.0,,,,,,,,, +611,28675,611,0.002962599695743052,1.0,1.0,,,,,,,,, +612,28700,612,0.0035731438272722468,1.0,1.0,,,,,,,,, +613,28781,613,0.012750156347203757,1.0,1.0,,,,,,,,, +614,28894,614,0.04075062133523626,1.0,1.0,,,,,,,,, +615,28935,615,0.0025724531672885863,1.0,1.0,,,,,,,,, +616,28960,616,0.0016636215113021535,1.0,1.0,,,,,,,,, +617,29049,617,0.0031745547367716593,1.0,1.0,0.023367196398148285,1.0,1.0,,,,,, +618,29074,618,0.001677857279321415,1.0,1.0,,,,,,,,, +619,29115,619,0.003383217667768804,1.0,1.0,,,,,,,,, +620,29188,620,0.028924054000064342,1.0,1.0,,,,,,,,, +621,29413,621,0.02791685022120451,1.0,1.0,,,,,,,,, +622,29534,622,0.03817232866789315,1.0,1.0,,,,,,,,, +623,29567,623,0.002411996990649641,1.0,1.0,,,,,,,,, +624,29592,624,0.003964523364211344,1.0,1.0,,,,,,,,, +625,29681,625,0.0030841385877267225,1.0,1.0,0.027998152176969146,1.0,1.0,,,,,, +626,29722,626,0.002293454506116632,1.0,1.0,,,,,,,,, +627,29763,627,0.005469170892400826,1.0,1.0,,,,,,,,, +628,29804,628,0.002298949975292544,1.0,1.0,,,,,,,,, +629,29845,629,0.0034687674441392746,1.0,1.0,,,,,,,,, +630,29878,630,0.002530964562582483,1.0,1.0,,,,,,,,, +631,29911,631,0.004269581710612016,1.0,1.0,,,,,,,,, +632,29952,632,0.005287030302905578,1.0,1.0,,,,,,,,, +633,29985,633,0.0033607394386157806,1.0,1.0,0.0026687447445783623,1.0,1.0,,,,,, +634,30010,634,0.0014693371871582408,1.0,1.0,,,,,,,,, +635,30043,635,0.0031539440315046367,1.0,1.0,,,,,,,,, +636,30076,636,0.0030359581355645557,1.0,1.0,,,,,,,,, +637,30101,637,0.0014814399549914015,1.0,1.0,,,,,,,,, +638,30142,638,0.0038873985496336444,1.0,1.0,,,,,,,,, +639,30175,639,0.0024085507500024054,1.0,1.0,,,,,,,,, +640,30208,640,0.0021334698368809767,1.0,1.0,,,,,,,,, +641,30249,641,0.003652064207110925,1.0,1.0,0.006725433730568967,1.0,1.0,,,,,, +642,30314,642,0.03416673927120598,0.0,0.0,,,,,,,,, +643,30355,643,0.004086532553790122,1.0,1.0,,,,,,,,, +644,30388,644,0.002299366494357855,1.0,1.0,,,,,,,,, +645,30413,645,0.0017982698780302068,1.0,1.0,,,,,,,,, +646,30438,646,0.0018251007819048938,1.0,1.0,,,,,,,,, +647,30471,647,0.0036919359401849762,1.0,1.0,,,,,,,,, +648,30496,648,0.00561342578629592,1.0,1.0,,,,,,,,, +649,30585,649,0.0025224411052725128,1.0,1.0,0.002001723699677314,1.0,1.0,,,,,, +650,30610,650,0.0014361863520245303,1.0,1.0,,,,,,,,, +651,30643,651,0.006170663640339171,1.0,1.0,,,,,,,,, +652,30684,652,0.004581734099439473,1.0,1.0,,,,,,,,, +653,30709,653,0.002163913224113306,1.0,1.0,,,,,,,,, +654,30750,654,0.0033945433870435416,1.0,1.0,,,,,,,,, +655,30775,655,0.002131028260448223,1.0,1.0,,,,,,,,, +656,30808,656,0.0037039272214381925,1.0,1.0,,,,,,,,, +657,30833,657,0.001974822690259259,1.0,1.0,0.0028021581104828784,1.0,1.0,,,,,, +658,30866,658,0.002156632485144314,1.0,1.0,,,,,,,,, +659,30891,659,0.0017080097990033968,1.0,1.0,,,,,,,,, +660,31012,660,0.038177843334056896,1.0,1.0,,,,,,,,, +661,31077,661,0.002747984222459224,1.0,1.0,,,,,,,,, +662,31166,662,0.0023493104742224923,1.0,1.0,,,,,,,,, +663,31191,663,0.008487881945835244,1.0,1.0,,,,,,,,, +664,31304,664,0.04231206878254235,1.0,1.0,,,,,,,,, +665,31361,665,0.015215694230513626,1.0,1.0,0.0013837619587805724,1.0,1.0,,,,,, +666,31490,666,0.03591513582745535,1.0,1.0,,,,,,,,, +667,31531,667,0.005085019459517483,1.0,1.0,,,,,,,,, +668,31556,668,0.001795864781023132,1.0,1.0,,,,,,,,, +669,31677,669,0.038666528565079405,1.0,1.0,,,,,,,,, +670,31710,670,0.0021584980261866345,1.0,1.0,,,,,,,,, +671,31743,671,0.008085111600470523,1.0,1.0,,,,,,,,, +672,31776,672,0.004535103148454738,1.0,1.0,,,,,,,,, +673,31809,673,0.0058906742437295646,1.0,1.0,0.006498069727926179,1.0,1.0,,,,,, +674,31834,674,0.001827985430524753,1.0,1.0,,,,,,,,, +675,31875,675,0.00364233891494115,1.0,1.0,,,,,,,,, +676,31900,676,0.001931552588011319,1.0,1.0,,,,,,,,, +677,32029,677,0.036205323013049476,1.0,1.0,,,,,,,,, +678,32054,678,0.0013846837102050812,1.0,1.0,,,,,,,,, +679,32095,679,0.0043542447196439306,1.0,1.0,,,,,,,,, +680,32120,680,0.00577771863262645,1.0,1.0,,,,,,,,, +681,32145,681,0.003957892355567859,1.0,1.0,0.021329540312391832,1.0,1.0,,,,,, +682,32186,682,0.0029775799205433353,1.0,1.0,,,,,,,,, +683,32259,683,0.027764181659045383,1.0,1.0,,,,,,,,, +684,32292,684,0.009632070107329582,1.0,1.0,,,,,,,,, +685,32317,685,0.001913364828647085,1.0,1.0,,,,,,,,, +686,32350,686,0.008256996994122261,1.0,1.0,,,,,,,,, +687,32439,687,0.002696479044664722,1.0,1.0,,,,,,,,, +688,32480,688,0.011437369231234697,1.0,1.0,,,,,,,,, +689,32521,689,0.004123384450207596,1.0,1.0,0.0013611759538904758,1.0,1.0,,,,,, +690,32554,690,0.021328601403353388,1.0,1.0,,,,,,,,, +691,32579,691,0.0019112908669925825,1.0,1.0,,,,,,,,, +692,32612,692,0.0020984052778896967,1.0,1.0,,,,,,,,, +693,32653,693,0.003886208962380579,1.0,1.0,,,,,,,,, +694,32734,694,0.02024900971205125,1.0,1.0,,,,,,,,, +695,32775,695,0.004149458056376716,1.0,1.0,,,,,,,,, +696,32800,696,0.007904861976567947,1.0,1.0,,,,,,,,, +697,32833,697,0.0030053626542559182,1.0,1.0,0.014331867724937788,1.0,1.0,,,,,, +698,32954,698,0.03846804843888004,1.0,1.0,,,,,,,,, +699,32987,699,0.004124619444397126,1.0,1.0,,,,,,,,, +700,33012,700,0.004192819879951155,1.0,1.0,,,,,,,,, +701,33037,701,0.005743955528466434,1.0,1.0,,,,,,,,, +702,33078,702,0.001999935150157285,1.0,1.0,,,,,,,,, +703,33111,703,0.008503282630630622,1.0,1.0,,,,,,,,, +704,33152,704,0.0037001522724915085,1.0,1.0,,,,,,,,, +705,33177,705,0.003087129954659531,1.0,1.0,0.012496777327306747,1.0,1.0,,,,,, +706,33210,706,0.00350194301292305,1.0,1.0,,,,,,,,, +707,33235,707,0.014080687296673014,1.0,1.0,,,,,,,,, +708,33276,708,0.00410933899247948,1.0,1.0,,,,,,,,, +709,33397,709,0.037851613223152565,1.0,1.0,,,,,,,,, +710,33510,710,0.040810033148644446,1.0,1.0,,,,,,,,, +711,33551,711,0.002418716375742591,1.0,1.0,,,,,,,,, +712,33592,712,0.003624795394367439,1.0,1.0,,,,,,,,, +713,33633,713,0.0037505140744567597,1.0,1.0,0.014227452131376592,1.0,1.0,,,,,, +714,33666,714,0.002089423445002012,1.0,1.0,,,,,,,,, +715,33771,715,0.025000065846041846,1.0,1.0,,,,,,,,, +716,33812,716,0.004941091954005515,1.0,1.0,,,,,,,,, +717,33845,717,0.0027349276239888812,1.0,1.0,,,,,,,,, +718,33886,718,0.005136407480595339,1.0,1.0,,,,,,,,, +719,33919,719,0.0019504698773107519,1.0,1.0,,,,,,,,, +720,33952,720,0.0019459015358324907,1.0,1.0,,,,,,,,, +721,33993,721,0.00907403369688086,1.0,1.0,0.003248707907472029,1.0,1.0,,,,,, +722,34018,722,0.0018039287319637268,1.0,1.0,,,,,,,,, +723,34043,723,0.0038322164314018884,1.0,1.0,,,,,,,,, +724,34108,724,0.0026914926717658714,1.0,1.0,,,,,,,,, +725,34229,725,0.038918225287749464,1.0,1.0,,,,,,,,, +726,34270,726,0.0032807265786382236,1.0,1.0,,,,,,,,, +727,34383,727,0.04014369331250983,1.0,1.0,,,,,,,,, +728,34424,728,0.0024898565655033598,1.0,1.0,,,,,,,,, +729,34457,729,0.0020895211450579153,1.0,1.0,0.0018949493887492016,1.0,1.0,,,,,, +730,34570,730,0.04094184044472123,1.0,1.0,,,,,,,,, +731,34595,731,0.004170812957869808,1.0,1.0,,,,,,,,, +732,34620,732,0.002500022928001897,1.0,1.0,,,,,,,,, +733,34661,733,0.0037272271053777786,1.0,1.0,,,,0.97826087474823,0.9714285714285714,1.0,0.9975490196078431,0.999147305981216,0.9855072463768115 + diff --git a/README.md b/README.md index 0322a27..e5be65a 100644 --- a/README.md +++ b/README.md @@ -35,7 +35,7 @@ or in BibTeX --- -## Table of Contents (Add the draft (in progress): AI for Forensics - Identifying IPs with a Fine-tuned Model, 8/2023) +## Table of Contents (Added: AI for Forensics - Identifying IPs with a Fine-tuned Model, 8/2023) - Basic Computer Skills for Digital Forensics