{ "cells": [ { "cell_type": "code", "execution_count": 3, "metadata": {}, "outputs": [], "source": [ "import sagemaker\n", "import boto3\n", "from sagemaker import get_execution_role\n", "\n", "boto_session = boto3.Session(profile_name='bonitoo', region_name='eu-central-1')\n", "sagemaker_session = sagemaker.LocalSession(boto_session=boto_session)\n", "#sagemaker_session = sagemaker.Session()" ] }, { "cell_type": "code", "execution_count": 4, "metadata": {}, "outputs": [], "source": [ "# Get a SageMaker-compatible role used by this Notebook Instance.\n", "#role = get_execution_role()\n", "role = 'Bonitoo_SageMaker_Execution'\n", "region = boto_session.region_name" ] }, { "cell_type": "code", "execution_count": 33, "metadata": {}, "outputs": [], "source": [ "train_input = 's3://customers-bonitoo-cachettl/sagemaker/data/export.csv'" ] }, { "cell_type": "code", "execution_count": 77, "metadata": {}, "outputs": [], "source": [ "from sagemaker.xgboost.estimator import XGBoost\n", "\n", "tf = XGBoost(\n", " entry_point='train_model.py',\n", " source_dir='./src',\n", " train_instance_type='local',\n", " train_instance_count=1,\n", " role=role,\n", " sagemaker_session=sagemaker_session,\n", " framework_version='0.90-1',\n", " py_version='py3',\n", " hyperparameters={\n", " 'bonitoo_price_pos_abs': 1000,\n", " 'bonitoo_price_neg_abs': 200,\n", " 'bonitoo_price_pos_perc': 0.05,\n", " 'bonitoo_price_neg_perc': 0.05,\n", " 'num_round': 20,\n", " 'max_depth': 15,\n", " 'eta': 0.5,\n", " 'num_class': 8,\n", " 'objective': 'multi:softprob',\n", " 'eval_metric': 'mlogloss'\n", " })" ] }, { "cell_type": "code", "execution_count": 78, "metadata": { "scrolled": true }, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "Creating tmpsn7kurwo_algo-1-hibva_1 ... \n", "\u001b[1BAttaching to tmpsn7kurwo_algo-1-hibva_12mdone\u001b[0m\n", "\u001b[36malgo-1-hibva_1 |\u001b[0m INFO:sagemaker-containers:Imported framework sagemaker_xgboost_container.training\n", "\u001b[36malgo-1-hibva_1 |\u001b[0m INFO:sagemaker-containers:No GPUs detected (normal if no gpus installed)\n", "\u001b[36malgo-1-hibva_1 |\u001b[0m INFO:sagemaker_xgboost_container.training:Invoking user training script.\n", "\u001b[36malgo-1-hibva_1 |\u001b[0m INFO:sagemaker-containers:Module train_model does not provide a setup.py. \n", "\u001b[36malgo-1-hibva_1 |\u001b[0m Generating setup.py\n", "\u001b[36malgo-1-hibva_1 |\u001b[0m INFO:sagemaker-containers:Generating setup.cfg\n", "\u001b[36malgo-1-hibva_1 |\u001b[0m INFO:sagemaker-containers:Generating MANIFEST.in\n", "\u001b[36malgo-1-hibva_1 |\u001b[0m INFO:sagemaker-containers:Installing module with the following command:\n", "\u001b[36malgo-1-hibva_1 |\u001b[0m /miniconda3/bin/python -m pip install . -r requirements.txt\n", "\u001b[36malgo-1-hibva_1 |\u001b[0m Processing /opt/ml/code\n", "\u001b[36malgo-1-hibva_1 |\u001b[0m Requirement already satisfied: pandas in /miniconda3/lib/python3.7/site-packages (from -r requirements.txt (line 1)) (0.25.1)\n", "\u001b[36malgo-1-hibva_1 |\u001b[0m Requirement already satisfied: numpy in /miniconda3/lib/python3.7/site-packages (from -r requirements.txt (line 2)) (1.17.2)\n", "\u001b[36malgo-1-hibva_1 |\u001b[0m Requirement already satisfied: python-dateutil>=2.6.1 in /miniconda3/lib/python3.7/site-packages (from pandas->-r requirements.txt (line 1)) (2.8.0)\n", "\u001b[36malgo-1-hibva_1 |\u001b[0m Requirement already satisfied: pytz>=2017.2 in /miniconda3/lib/python3.7/site-packages (from pandas->-r requirements.txt (line 1)) (2019.3)\n", "\u001b[36malgo-1-hibva_1 |\u001b[0m Requirement already satisfied: six>=1.5 in /miniconda3/lib/python3.7/site-packages (from python-dateutil>=2.6.1->pandas->-r requirements.txt (line 1)) (1.12.0)\n", "\u001b[36malgo-1-hibva_1 |\u001b[0m Building wheels for collected packages: train-model\n", "\u001b[36malgo-1-hibva_1 |\u001b[0m Building wheel for train-model (setup.py) ... \u001b[?25ldone\n", "\u001b[36malgo-1-hibva_1 |\u001b[0m \u001b[?25h Created wheel for train-model: filename=train_model-1.0.0-py2.py3-none-any.whl size=12596 sha256=1e1372c49fcc19ef6d93ad652d2e5c79e5855068be011b19b2273a3aff1b098f\n", "\u001b[36malgo-1-hibva_1 |\u001b[0m Stored in directory: /tmp/pip-ephem-wheel-cache-yjganydo/wheels/35/24/16/37574d11bf9bde50616c67372a334f94fa8356bc7164af8ca3\n", "\u001b[36malgo-1-hibva_1 |\u001b[0m Successfully built train-model\n", "\u001b[36malgo-1-hibva_1 |\u001b[0m Installing collected packages: train-model\n", "\u001b[36malgo-1-hibva_1 |\u001b[0m Successfully installed train-model-1.0.0\n", "\u001b[36malgo-1-hibva_1 |\u001b[0m INFO:sagemaker-containers:No GPUs detected (normal if no gpus installed)\n", "\u001b[36malgo-1-hibva_1 |\u001b[0m INFO:sagemaker-containers:Invoking user script\n", "\u001b[36malgo-1-hibva_1 |\u001b[0m \n", "\u001b[36malgo-1-hibva_1 |\u001b[0m Training Env:\n", "\u001b[36malgo-1-hibva_1 |\u001b[0m \n", "\u001b[36malgo-1-hibva_1 |\u001b[0m {\n", "\u001b[36malgo-1-hibva_1 |\u001b[0m \"additional_framework_parameters\": {},\n", "\u001b[36malgo-1-hibva_1 |\u001b[0m \"channel_input_dirs\": {\n", "\u001b[36malgo-1-hibva_1 |\u001b[0m \"training\": \"/opt/ml/input/data/training\"\n", "\u001b[36malgo-1-hibva_1 |\u001b[0m },\n", "\u001b[36malgo-1-hibva_1 |\u001b[0m \"current_host\": \"algo-1-hibva\",\n", "\u001b[36malgo-1-hibva_1 |\u001b[0m \"framework_module\": \"sagemaker_xgboost_container.training:main\",\n", "\u001b[36malgo-1-hibva_1 |\u001b[0m \"hosts\": [\n", "\u001b[36malgo-1-hibva_1 |\u001b[0m \"algo-1-hibva\"\n", "\u001b[36malgo-1-hibva_1 |\u001b[0m ],\n", "\u001b[36malgo-1-hibva_1 |\u001b[0m \"hyperparameters\": {\n", "\u001b[36malgo-1-hibva_1 |\u001b[0m \"bonitoo_price_pos_abs\": 1000,\n", "\u001b[36malgo-1-hibva_1 |\u001b[0m \"bonitoo_price_neg_abs\": 200,\n", "\u001b[36malgo-1-hibva_1 |\u001b[0m \"bonitoo_price_pos_perc\": 0.05,\n", "\u001b[36malgo-1-hibva_1 |\u001b[0m \"bonitoo_price_neg_perc\": 0.05,\n", "\u001b[36malgo-1-hibva_1 |\u001b[0m \"num_round\": 20,\n", "\u001b[36malgo-1-hibva_1 |\u001b[0m \"max_depth\": 15,\n", "\u001b[36malgo-1-hibva_1 |\u001b[0m \"eta\": 0.5,\n", "\u001b[36malgo-1-hibva_1 |\u001b[0m \"num_class\": 8,\n", "\u001b[36malgo-1-hibva_1 |\u001b[0m \"objective\": \"multi:softprob\",\n", "\u001b[36malgo-1-hibva_1 |\u001b[0m \"eval_metric\": \"mlogloss\"\n", "\u001b[36malgo-1-hibva_1 |\u001b[0m },\n", "\u001b[36malgo-1-hibva_1 |\u001b[0m \"input_config_dir\": \"/opt/ml/input/config\",\n", "\u001b[36malgo-1-hibva_1 |\u001b[0m \"input_data_config\": {\n", "\u001b[36malgo-1-hibva_1 |\u001b[0m \"training\": {\n", "\u001b[36malgo-1-hibva_1 |\u001b[0m \"TrainingInputMode\": \"File\"\n", "\u001b[36malgo-1-hibva_1 |\u001b[0m }\n", "\u001b[36malgo-1-hibva_1 |\u001b[0m },\n", "\u001b[36malgo-1-hibva_1 |\u001b[0m \"input_dir\": \"/opt/ml/input\",\n", "\u001b[36malgo-1-hibva_1 |\u001b[0m \"is_master\": true,\n", "\u001b[36malgo-1-hibva_1 |\u001b[0m \"job_name\": \"sagemaker-xgboost-2019-10-19-17-27-30-738\",\n", "\u001b[36malgo-1-hibva_1 |\u001b[0m \"log_level\": 20,\n", "\u001b[36malgo-1-hibva_1 |\u001b[0m \"master_hostname\": \"algo-1-hibva\",\n", "\u001b[36malgo-1-hibva_1 |\u001b[0m \"model_dir\": \"/opt/ml/model\",\n", "\u001b[36malgo-1-hibva_1 |\u001b[0m \"module_dir\": \"s3://sagemaker-eu-central-1-029917565482/sagemaker-xgboost-2019-10-19-17-27-30-738/source/sourcedir.tar.gz\",\n", "\u001b[36malgo-1-hibva_1 |\u001b[0m \"module_name\": \"train_model\",\n", "\u001b[36malgo-1-hibva_1 |\u001b[0m \"network_interface_name\": \"eth0\",\n", "\u001b[36malgo-1-hibva_1 |\u001b[0m \"num_cpus\": 4,\n", "\u001b[36malgo-1-hibva_1 |\u001b[0m \"num_gpus\": 0,\n", "\u001b[36malgo-1-hibva_1 |\u001b[0m \"output_data_dir\": \"/opt/ml/output/data\",\n", "\u001b[36malgo-1-hibva_1 |\u001b[0m \"output_dir\": \"/opt/ml/output\",\n", "\u001b[36malgo-1-hibva_1 |\u001b[0m \"output_intermediate_dir\": \"/opt/ml/output/intermediate\",\n", "\u001b[36malgo-1-hibva_1 |\u001b[0m \"resource_config\": {\n", "\u001b[36malgo-1-hibva_1 |\u001b[0m \"current_host\": \"algo-1-hibva\",\n", "\u001b[36malgo-1-hibva_1 |\u001b[0m \"hosts\": [\n", "\u001b[36malgo-1-hibva_1 |\u001b[0m \"algo-1-hibva\"\n", "\u001b[36malgo-1-hibva_1 |\u001b[0m ]\n", "\u001b[36malgo-1-hibva_1 |\u001b[0m },\n", "\u001b[36malgo-1-hibva_1 |\u001b[0m \"user_entry_point\": \"train_model.py\"\n", "\u001b[36malgo-1-hibva_1 |\u001b[0m }\n", "\u001b[36malgo-1-hibva_1 |\u001b[0m \n", "\u001b[36malgo-1-hibva_1 |\u001b[0m Environment variables:\n", "\u001b[36malgo-1-hibva_1 |\u001b[0m \n", "\u001b[36malgo-1-hibva_1 |\u001b[0m SM_HOSTS=[\"algo-1-hibva\"]\n", "\u001b[36malgo-1-hibva_1 |\u001b[0m SM_NETWORK_INTERFACE_NAME=eth0\n", "\u001b[36malgo-1-hibva_1 |\u001b[0m SM_HPS={\"bonitoo_price_neg_abs\":200,\"bonitoo_price_neg_perc\":0.05,\"bonitoo_price_pos_abs\":1000,\"bonitoo_price_pos_perc\":0.05,\"eta\":0.5,\"eval_metric\":\"mlogloss\",\"max_depth\":15,\"num_class\":8,\"num_round\":20,\"objective\":\"multi:softprob\"}\n", "\u001b[36malgo-1-hibva_1 |\u001b[0m SM_USER_ENTRY_POINT=train_model.py\n", "\u001b[36malgo-1-hibva_1 |\u001b[0m SM_FRAMEWORK_PARAMS={}\n", "\u001b[36malgo-1-hibva_1 |\u001b[0m SM_RESOURCE_CONFIG={\"current_host\":\"algo-1-hibva\",\"hosts\":[\"algo-1-hibva\"]}\n", "\u001b[36malgo-1-hibva_1 |\u001b[0m SM_INPUT_DATA_CONFIG={\"training\":{\"TrainingInputMode\":\"File\"}}\n", "\u001b[36malgo-1-hibva_1 |\u001b[0m SM_OUTPUT_DATA_DIR=/opt/ml/output/data\n", "\u001b[36malgo-1-hibva_1 |\u001b[0m SM_CHANNELS=[\"training\"]\n", "\u001b[36malgo-1-hibva_1 |\u001b[0m SM_CURRENT_HOST=algo-1-hibva\n", "\u001b[36malgo-1-hibva_1 |\u001b[0m SM_MODULE_NAME=train_model\n", "\u001b[36malgo-1-hibva_1 |\u001b[0m SM_LOG_LEVEL=20\n", "\u001b[36malgo-1-hibva_1 |\u001b[0m SM_FRAMEWORK_MODULE=sagemaker_xgboost_container.training:main\n", "\u001b[36malgo-1-hibva_1 |\u001b[0m SM_INPUT_DIR=/opt/ml/input\n", "\u001b[36malgo-1-hibva_1 |\u001b[0m SM_INPUT_CONFIG_DIR=/opt/ml/input/config\n", "\u001b[36malgo-1-hibva_1 |\u001b[0m SM_OUTPUT_DIR=/opt/ml/output\n", "\u001b[36malgo-1-hibva_1 |\u001b[0m SM_NUM_CPUS=4\n", "\u001b[36malgo-1-hibva_1 |\u001b[0m SM_NUM_GPUS=0\n", "\u001b[36malgo-1-hibva_1 |\u001b[0m SM_MODEL_DIR=/opt/ml/model\n", "\u001b[36malgo-1-hibva_1 |\u001b[0m SM_MODULE_DIR=s3://sagemaker-eu-central-1-029917565482/sagemaker-xgboost-2019-10-19-17-27-30-738/source/sourcedir.tar.gz\n", "\u001b[36malgo-1-hibva_1 |\u001b[0m SM_TRAINING_ENV={\"additional_framework_parameters\":{},\"channel_input_dirs\":{\"training\":\"/opt/ml/input/data/training\"},\"current_host\":\"algo-1-hibva\",\"framework_module\":\"sagemaker_xgboost_container.training:main\",\"hosts\":[\"algo-1-hibva\"],\"hyperparameters\":{\"bonitoo_price_neg_abs\":200,\"bonitoo_price_neg_perc\":0.05,\"bonitoo_price_pos_abs\":1000,\"bonitoo_price_pos_perc\":0.05,\"eta\":0.5,\"eval_metric\":\"mlogloss\",\"max_depth\":15,\"num_class\":8,\"num_round\":20,\"objective\":\"multi:softprob\"},\"input_config_dir\":\"/opt/ml/input/config\",\"input_data_config\":{\"training\":{\"TrainingInputMode\":\"File\"}},\"input_dir\":\"/opt/ml/input\",\"is_master\":true,\"job_name\":\"sagemaker-xgboost-2019-10-19-17-27-30-738\",\"log_level\":20,\"master_hostname\":\"algo-1-hibva\",\"model_dir\":\"/opt/ml/model\",\"module_dir\":\"s3://sagemaker-eu-central-1-029917565482/sagemaker-xgboost-2019-10-19-17-27-30-738/source/sourcedir.tar.gz\",\"module_name\":\"train_model\",\"network_interface_name\":\"eth0\",\"num_cpus\":4,\"num_gpus\":0,\"output_data_dir\":\"/opt/ml/output/data\",\"output_dir\":\"/opt/ml/output\",\"output_intermediate_dir\":\"/opt/ml/output/intermediate\",\"resource_config\":{\"current_host\":\"algo-1-hibva\",\"hosts\":[\"algo-1-hibva\"]},\"user_entry_point\":\"train_model.py\"}\n", "\u001b[36malgo-1-hibva_1 |\u001b[0m SM_USER_ARGS=[\"--bonitoo_price_neg_abs\",\"200\",\"--bonitoo_price_neg_perc\",\"0.05\",\"--bonitoo_price_pos_abs\",\"1000\",\"--bonitoo_price_pos_perc\",\"0.05\",\"--eta\",\"0.5\",\"--eval_metric\",\"mlogloss\",\"--max_depth\",\"15\",\"--num_class\",\"8\",\"--num_round\",\"20\",\"--objective\",\"multi:softprob\"]\n", "\u001b[36malgo-1-hibva_1 |\u001b[0m SM_OUTPUT_INTERMEDIATE_DIR=/opt/ml/output/intermediate\n", "\u001b[36malgo-1-hibva_1 |\u001b[0m SM_CHANNEL_TRAINING=/opt/ml/input/data/training\n", "\u001b[36malgo-1-hibva_1 |\u001b[0m SM_HP_BONITOO_PRICE_POS_ABS=1000\n", "\u001b[36malgo-1-hibva_1 |\u001b[0m SM_HP_BONITOO_PRICE_NEG_ABS=200\n", "\u001b[36malgo-1-hibva_1 |\u001b[0m SM_HP_BONITOO_PRICE_POS_PERC=0.05\n", "\u001b[36malgo-1-hibva_1 |\u001b[0m SM_HP_BONITOO_PRICE_NEG_PERC=0.05\n", "\u001b[36malgo-1-hibva_1 |\u001b[0m SM_HP_NUM_ROUND=20\n", "\u001b[36malgo-1-hibva_1 |\u001b[0m SM_HP_MAX_DEPTH=15\n", "\u001b[36malgo-1-hibva_1 |\u001b[0m SM_HP_ETA=0.5\n", "\u001b[36malgo-1-hibva_1 |\u001b[0m SM_HP_NUM_CLASS=8\n", "\u001b[36malgo-1-hibva_1 |\u001b[0m SM_HP_OBJECTIVE=multi:softprob\n", "\u001b[36malgo-1-hibva_1 |\u001b[0m SM_HP_EVAL_METRIC=mlogloss\n", "\u001b[36malgo-1-hibva_1 |\u001b[0m PYTHONPATH=/miniconda3/bin:/:/usr/local/lib/python3.5/dist-packages/xgboost/dmlc-core/tracker:/miniconda3/lib/python37.zip:/miniconda3/lib/python3.7:/miniconda3/lib/python3.7/lib-dynload:/miniconda3/lib/python3.7/site-packages\n", "\u001b[36malgo-1-hibva_1 |\u001b[0m \n", "\u001b[36malgo-1-hibva_1 |\u001b[0m Invoking script with the following command:\n", "\u001b[36malgo-1-hibva_1 |\u001b[0m \n", "\u001b[36malgo-1-hibva_1 |\u001b[0m /miniconda3/bin/python -m train_model --bonitoo_price_neg_abs 200 --bonitoo_price_neg_perc 0.05 --bonitoo_price_pos_abs 1000 --bonitoo_price_pos_perc 0.05 --eta 0.5 --eval_metric mlogloss --max_depth 15 --num_class 8 --num_round 20 --objective multi:softprob\n", "\u001b[36malgo-1-hibva_1 |\u001b[0m \n", "\u001b[36malgo-1-hibva_1 |\u001b[0m \n" ] }, { "name": "stdout", "output_type": "stream", "text": [ "\u001b[36malgo-1-hibva_1 |\u001b[0m [0]\ttrain-mlogloss:0.848017\tvalidation-mlogloss:0.922091\n", "\u001b[36malgo-1-hibva_1 |\u001b[0m [1]\ttrain-mlogloss:0.578424\tvalidation-mlogloss:0.697124\n", "\u001b[36malgo-1-hibva_1 |\u001b[0m [2]\ttrain-mlogloss:0.419099\tvalidation-mlogloss:0.572552\n", "\u001b[36malgo-1-hibva_1 |\u001b[0m [3]\ttrain-mlogloss:0.31692\tvalidation-mlogloss:0.497591\n", "\u001b[36malgo-1-hibva_1 |\u001b[0m [4]\ttrain-mlogloss:0.247843\tvalidation-mlogloss:0.450857\n", "\u001b[36malgo-1-hibva_1 |\u001b[0m [5]\ttrain-mlogloss:0.20313\tvalidation-mlogloss:0.42247\n", "\u001b[36malgo-1-hibva_1 |\u001b[0m [6]\ttrain-mlogloss:0.171749\tvalidation-mlogloss:0.404928\n", "\u001b[36malgo-1-hibva_1 |\u001b[0m [7]\ttrain-mlogloss:0.15009\tvalidation-mlogloss:0.393772\n", "\u001b[36malgo-1-hibva_1 |\u001b[0m [8]\ttrain-mlogloss:0.133377\tvalidation-mlogloss:0.385623\n", "\u001b[36malgo-1-hibva_1 |\u001b[0m [9]\ttrain-mlogloss:0.120209\tvalidation-mlogloss:0.378456\n", "\u001b[36malgo-1-hibva_1 |\u001b[0m [10]\ttrain-mlogloss:0.110155\tvalidation-mlogloss:0.374374\n", "\u001b[36malgo-1-hibva_1 |\u001b[0m [11]\ttrain-mlogloss:0.09938\tvalidation-mlogloss:0.36958\n", "\u001b[36malgo-1-hibva_1 |\u001b[0m [12]\ttrain-mlogloss:0.092882\tvalidation-mlogloss:0.366292\n", "\u001b[36malgo-1-hibva_1 |\u001b[0m [13]\ttrain-mlogloss:0.085552\tvalidation-mlogloss:0.363469\n", "\u001b[36malgo-1-hibva_1 |\u001b[0m [14]\ttrain-mlogloss:0.079976\tvalidation-mlogloss:0.363688\n", "\u001b[36malgo-1-hibva_1 |\u001b[0m [15]\ttrain-mlogloss:0.075524\tvalidation-mlogloss:0.36325\n", "\u001b[36malgo-1-hibva_1 |\u001b[0m [16]\ttrain-mlogloss:0.069857\tvalidation-mlogloss:0.36269\n", "\u001b[36malgo-1-hibva_1 |\u001b[0m [17]\ttrain-mlogloss:0.065141\tvalidation-mlogloss:0.361854\n", "\u001b[36malgo-1-hibva_1 |\u001b[0m [18]\ttrain-mlogloss:0.062355\tvalidation-mlogloss:0.361638\n", "\u001b[36malgo-1-hibva_1 |\u001b[0m [19]\ttrain-mlogloss:0.060227\tvalidation-mlogloss:0.361047\n", "\u001b[36mtmpsn7kurwo_algo-1-hibva_1 exited with code 0\n", "\u001b[0mAborting on container exit...\n", "===== Job Complete =====\n" ] } ], "source": [ "estimator = tf.fit({'training': train_input})\n", "#estimator = sklearn.attach('sagemaker-scikit-learn-2019-01-25-16-34-38-829')" ] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": [] } ], "metadata": { "kernelspec": { "display_name": "Python 3", "language": "python", "name": "python3" }, "language_info": { "codemirror_mode": { "name": "ipython", "version": 3 }, "file_extension": ".py", "mimetype": "text/x-python", "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", "version": "3.7.3" } }, "nbformat": 4, "nbformat_minor": 2 }