{ "cells": [ { "cell_type": "code", "execution_count": 1, "metadata": {}, "outputs": [], "source": [ "import sagemaker\n", "import boto3\n", "from sagemaker import get_execution_role\n", "\n", "boto_session = boto3.Session(profile_name='bonitoo', region_name='eu-central-1')\n", "sagemaker_session = sagemaker.LocalSession(boto_session=boto_session)\n", "#sagemaker_session = sagemaker.Session()" ] }, { "cell_type": "code", "execution_count": 2, "metadata": {}, "outputs": [], "source": [ "# Get a SageMaker-compatible role used by this Notebook Instance.\n", "#role = get_execution_role()\n", "role = 'Bonitoo_SageMaker_Execution'\n", "region = boto_session.region_name" ] }, { "cell_type": "code", "execution_count": 3, "metadata": {}, "outputs": [], "source": [ "train_input = 's3://customers-bonitoo-cachettl/sagemaker/data/export.csv'" ] }, { "cell_type": "code", "execution_count": 50, "metadata": {}, "outputs": [], "source": [ "from sagemaker.xgboost.estimator import XGBoost\n", "\n", "tf = XGBoost(\n", " entry_point='train_model.py',\n", " source_dir='./src',\n", " train_instance_type='local',\n", " train_instance_count=1,\n", " role=role,\n", " sagemaker_session=sagemaker_session,\n", " framework_version='0.90-1',\n", " py_version='py3',\n", " hyperparameters={\n", " 'bonitoo_price_limit': 1000,\n", " 'num_round': 15,\n", " 'max_depth': 15,\n", " 'eta': 0.5,\n", " 'num_class': 8,\n", " 'objective': 'multi:softmax',\n", " 'eval_metric': 'mlogloss'\n", " })" ] }, { "cell_type": "code", "execution_count": 51, "metadata": { "scrolled": true }, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "Creating tmptao5hpuc_algo-1-x6dhm_1 ... \n", "\u001b[1BAttaching to tmptao5hpuc_algo-1-x6dhm_12mdone\u001b[0m\n", "\u001b[36malgo-1-x6dhm_1 |\u001b[0m INFO:sagemaker-containers:Imported framework sagemaker_xgboost_container.training\n", "\u001b[36malgo-1-x6dhm_1 |\u001b[0m INFO:sagemaker-containers:No GPUs detected (normal if no gpus installed)\n", "\u001b[36malgo-1-x6dhm_1 |\u001b[0m INFO:sagemaker_xgboost_container.training:Invoking user training script.\n", "\u001b[36malgo-1-x6dhm_1 |\u001b[0m INFO:sagemaker-containers:Module train_model does not provide a setup.py. \n", "\u001b[36malgo-1-x6dhm_1 |\u001b[0m Generating setup.py\n", "\u001b[36malgo-1-x6dhm_1 |\u001b[0m INFO:sagemaker-containers:Generating setup.cfg\n", "\u001b[36malgo-1-x6dhm_1 |\u001b[0m INFO:sagemaker-containers:Generating MANIFEST.in\n", "\u001b[36malgo-1-x6dhm_1 |\u001b[0m INFO:sagemaker-containers:Installing module with the following command:\n", "\u001b[36malgo-1-x6dhm_1 |\u001b[0m /usr/bin/python3 -m pip install . -r requirements.txt\n", "\u001b[36malgo-1-x6dhm_1 |\u001b[0m Processing /opt/ml/code\n", "\u001b[36malgo-1-x6dhm_1 |\u001b[0m Requirement already satisfied: pandas in /usr/local/lib/python3.5/dist-packages (from -r requirements.txt (line 1)) (0.24.2)\n", "\u001b[36malgo-1-x6dhm_1 |\u001b[0m Requirement already satisfied: numpy in /usr/local/lib/python3.5/dist-packages (from -r requirements.txt (line 2)) (1.17.2)\n", "\u001b[36malgo-1-x6dhm_1 |\u001b[0m Requirement already satisfied: pytz>=2011k in /usr/local/lib/python3.5/dist-packages (from pandas->-r requirements.txt (line 1)) (2019.2)\n", "\u001b[36malgo-1-x6dhm_1 |\u001b[0m Requirement already satisfied: python-dateutil>=2.5.0 in /usr/local/lib/python3.5/dist-packages (from pandas->-r requirements.txt (line 1)) (2.8.0)\n", "\u001b[36malgo-1-x6dhm_1 |\u001b[0m Requirement already satisfied: six>=1.5 in /usr/local/lib/python3.5/dist-packages (from python-dateutil>=2.5.0->pandas->-r requirements.txt (line 1)) (1.12.0)\n", "\u001b[36malgo-1-x6dhm_1 |\u001b[0m Building wheels for collected packages: train-model\n", "\u001b[36malgo-1-x6dhm_1 |\u001b[0m Building wheel for train-model (setup.py) ... \u001b[?25ldone\n", "\u001b[36malgo-1-x6dhm_1 |\u001b[0m \u001b[?25h Created wheel for train-model: filename=train_model-1.0.0-py2.py3-none-any.whl size=6578 sha256=f2f4bac7a2d0260f534e32b3ac0341fb291f30669499adf59ead09aa62b7ccc5\n", "\u001b[36malgo-1-x6dhm_1 |\u001b[0m Stored in directory: /tmp/pip-ephem-wheel-cache-vdfjugbr/wheels/35/24/16/37574d11bf9bde50616c67372a334f94fa8356bc7164af8ca3\n", "\u001b[36malgo-1-x6dhm_1 |\u001b[0m Successfully built train-model\n", "\u001b[36malgo-1-x6dhm_1 |\u001b[0m Installing collected packages: train-model\n", "\u001b[36malgo-1-x6dhm_1 |\u001b[0m Successfully installed train-model-1.0.0\n", "\u001b[36malgo-1-x6dhm_1 |\u001b[0m INFO:sagemaker-containers:No GPUs detected (normal if no gpus installed)\n", "\u001b[36malgo-1-x6dhm_1 |\u001b[0m INFO:sagemaker-containers:Invoking user script\n", "\u001b[36malgo-1-x6dhm_1 |\u001b[0m \n", "\u001b[36malgo-1-x6dhm_1 |\u001b[0m Training Env:\n", "\u001b[36malgo-1-x6dhm_1 |\u001b[0m \n", "\u001b[36malgo-1-x6dhm_1 |\u001b[0m {\n", "\u001b[36malgo-1-x6dhm_1 |\u001b[0m \"network_interface_name\": \"eth0\",\n", "\u001b[36malgo-1-x6dhm_1 |\u001b[0m \"hosts\": [\n", "\u001b[36malgo-1-x6dhm_1 |\u001b[0m \"algo-1-x6dhm\"\n", "\u001b[36malgo-1-x6dhm_1 |\u001b[0m ],\n", "\u001b[36malgo-1-x6dhm_1 |\u001b[0m \"log_level\": 20,\n", "\u001b[36malgo-1-x6dhm_1 |\u001b[0m \"input_config_dir\": \"/opt/ml/input/config\",\n", "\u001b[36malgo-1-x6dhm_1 |\u001b[0m \"framework_module\": \"sagemaker_xgboost_container.training:main\",\n", "\u001b[36malgo-1-x6dhm_1 |\u001b[0m \"input_dir\": \"/opt/ml/input\",\n", "\u001b[36malgo-1-x6dhm_1 |\u001b[0m \"channel_input_dirs\": {\n", "\u001b[36malgo-1-x6dhm_1 |\u001b[0m \"training\": \"/opt/ml/input/data/training\"\n", "\u001b[36malgo-1-x6dhm_1 |\u001b[0m },\n", "\u001b[36malgo-1-x6dhm_1 |\u001b[0m \"num_gpus\": 0,\n", "\u001b[36malgo-1-x6dhm_1 |\u001b[0m \"job_name\": \"sagemaker-xgboost-2019-10-05-20-16-58-398\",\n", "\u001b[36malgo-1-x6dhm_1 |\u001b[0m \"output_intermediate_dir\": \"/opt/ml/output/intermediate\",\n", "\u001b[36malgo-1-x6dhm_1 |\u001b[0m \"user_entry_point\": \"train_model.py\",\n", "\u001b[36malgo-1-x6dhm_1 |\u001b[0m \"current_host\": \"algo-1-x6dhm\",\n", "\u001b[36malgo-1-x6dhm_1 |\u001b[0m \"module_dir\": \"s3://sagemaker-eu-central-1-029917565482/sagemaker-xgboost-2019-10-05-20-16-58-398/source/sourcedir.tar.gz\",\n", "\u001b[36malgo-1-x6dhm_1 |\u001b[0m \"master_hostname\": \"algo-1-x6dhm\",\n", "\u001b[36malgo-1-x6dhm_1 |\u001b[0m \"module_name\": \"train_model\",\n", "\u001b[36malgo-1-x6dhm_1 |\u001b[0m \"resource_config\": {\n", "\u001b[36malgo-1-x6dhm_1 |\u001b[0m \"current_host\": \"algo-1-x6dhm\",\n", "\u001b[36malgo-1-x6dhm_1 |\u001b[0m \"hosts\": [\n", "\u001b[36malgo-1-x6dhm_1 |\u001b[0m \"algo-1-x6dhm\"\n", "\u001b[36malgo-1-x6dhm_1 |\u001b[0m ]\n", "\u001b[36malgo-1-x6dhm_1 |\u001b[0m },\n", "\u001b[36malgo-1-x6dhm_1 |\u001b[0m \"additional_framework_parameters\": {},\n", "\u001b[36malgo-1-x6dhm_1 |\u001b[0m \"num_cpus\": 6,\n", "\u001b[36malgo-1-x6dhm_1 |\u001b[0m \"output_data_dir\": \"/opt/ml/output/data\",\n", "\u001b[36malgo-1-x6dhm_1 |\u001b[0m \"input_data_config\": {\n", "\u001b[36malgo-1-x6dhm_1 |\u001b[0m \"training\": {\n", "\u001b[36malgo-1-x6dhm_1 |\u001b[0m \"TrainingInputMode\": \"File\"\n", "\u001b[36malgo-1-x6dhm_1 |\u001b[0m }\n", "\u001b[36malgo-1-x6dhm_1 |\u001b[0m },\n", "\u001b[36malgo-1-x6dhm_1 |\u001b[0m \"is_master\": true,\n", "\u001b[36malgo-1-x6dhm_1 |\u001b[0m \"hyperparameters\": {\n", "\u001b[36malgo-1-x6dhm_1 |\u001b[0m \"bonitoo_price_limit\": 1000,\n", "\u001b[36malgo-1-x6dhm_1 |\u001b[0m \"max_depth\": 15,\n", "\u001b[36malgo-1-x6dhm_1 |\u001b[0m \"objective\": \"multi:softmax\",\n", "\u001b[36malgo-1-x6dhm_1 |\u001b[0m \"num_class\": 8,\n", "\u001b[36malgo-1-x6dhm_1 |\u001b[0m \"eta\": 0.5,\n", "\u001b[36malgo-1-x6dhm_1 |\u001b[0m \"eval_metric\": \"mlogloss\",\n", "\u001b[36malgo-1-x6dhm_1 |\u001b[0m \"num_round\": 15\n", "\u001b[36malgo-1-x6dhm_1 |\u001b[0m },\n", "\u001b[36malgo-1-x6dhm_1 |\u001b[0m \"output_dir\": \"/opt/ml/output\",\n", "\u001b[36malgo-1-x6dhm_1 |\u001b[0m \"model_dir\": \"/opt/ml/model\"\n", "\u001b[36malgo-1-x6dhm_1 |\u001b[0m }\n", "\u001b[36malgo-1-x6dhm_1 |\u001b[0m \n", "\u001b[36malgo-1-x6dhm_1 |\u001b[0m Environment variables:\n", "\u001b[36malgo-1-x6dhm_1 |\u001b[0m \n", "\u001b[36malgo-1-x6dhm_1 |\u001b[0m SM_INPUT_CONFIG_DIR=/opt/ml/input/config\n", "\u001b[36malgo-1-x6dhm_1 |\u001b[0m SM_HP_MAX_DEPTH=15\n", "\u001b[36malgo-1-x6dhm_1 |\u001b[0m SM_LOG_LEVEL=20\n", "\u001b[36malgo-1-x6dhm_1 |\u001b[0m SM_OUTPUT_DIR=/opt/ml/output\n", "\u001b[36malgo-1-x6dhm_1 |\u001b[0m SM_NUM_CPUS=6\n", "\u001b[36malgo-1-x6dhm_1 |\u001b[0m SM_CHANNELS=[\"training\"]\n", "\u001b[36malgo-1-x6dhm_1 |\u001b[0m SM_HP_NUM_ROUND=15\n", "\u001b[36malgo-1-x6dhm_1 |\u001b[0m SM_HP_OBJECTIVE=multi:softmax\n", "\u001b[36malgo-1-x6dhm_1 |\u001b[0m SM_OUTPUT_DATA_DIR=/opt/ml/output/data\n", "\u001b[36malgo-1-x6dhm_1 |\u001b[0m SM_FRAMEWORK_MODULE=sagemaker_xgboost_container.training:main\n", "\u001b[36malgo-1-x6dhm_1 |\u001b[0m SM_NETWORK_INTERFACE_NAME=eth0\n", "\u001b[36malgo-1-x6dhm_1 |\u001b[0m SM_INPUT_DATA_CONFIG={\"training\":{\"TrainingInputMode\":\"File\"}}\n", "\u001b[36malgo-1-x6dhm_1 |\u001b[0m SM_FRAMEWORK_PARAMS={}\n", "\u001b[36malgo-1-x6dhm_1 |\u001b[0m SM_HPS={\"bonitoo_price_limit\":1000,\"eta\":0.5,\"eval_metric\":\"mlogloss\",\"max_depth\":15,\"num_class\":8,\"num_round\":15,\"objective\":\"multi:softmax\"}\n", "\u001b[36malgo-1-x6dhm_1 |\u001b[0m PYTHONPATH=/usr/local/bin:/:/usr/local/lib/python3.5/dist-packages/xgboost/dmlc-core/tracker:/usr/lib/python35.zip:/usr/lib/python3.5:/usr/lib/python3.5/plat-x86_64-linux-gnu:/usr/lib/python3.5/lib-dynload:/usr/local/lib/python3.5/dist-packages:/usr/lib/python3/dist-packages\n", "\u001b[36malgo-1-x6dhm_1 |\u001b[0m SM_RESOURCE_CONFIG={\"current_host\":\"algo-1-x6dhm\",\"hosts\":[\"algo-1-x6dhm\"]}\n", "\u001b[36malgo-1-x6dhm_1 |\u001b[0m SM_NUM_GPUS=0\n", "\u001b[36malgo-1-x6dhm_1 |\u001b[0m SM_HP_ETA=0.5\n", "\u001b[36malgo-1-x6dhm_1 |\u001b[0m SM_HP_NUM_CLASS=8\n", "\u001b[36malgo-1-x6dhm_1 |\u001b[0m SM_MODULE_DIR=s3://sagemaker-eu-central-1-029917565482/sagemaker-xgboost-2019-10-05-20-16-58-398/source/sourcedir.tar.gz\n", "\u001b[36malgo-1-x6dhm_1 |\u001b[0m SM_USER_ARGS=[\"--bonitoo_price_limit\",\"1000\",\"--eta\",\"0.5\",\"--eval_metric\",\"mlogloss\",\"--max_depth\",\"15\",\"--num_class\",\"8\",\"--num_round\",\"15\",\"--objective\",\"multi:softmax\"]\n", "\u001b[36malgo-1-x6dhm_1 |\u001b[0m SM_USER_ENTRY_POINT=train_model.py\n", "\u001b[36malgo-1-x6dhm_1 |\u001b[0m SM_CURRENT_HOST=algo-1-x6dhm\n", "\u001b[36malgo-1-x6dhm_1 |\u001b[0m SM_INPUT_DIR=/opt/ml/input\n", "\u001b[36malgo-1-x6dhm_1 |\u001b[0m SM_CHANNEL_TRAINING=/opt/ml/input/data/training\n", "\u001b[36malgo-1-x6dhm_1 |\u001b[0m SM_OUTPUT_INTERMEDIATE_DIR=/opt/ml/output/intermediate\n", "\u001b[36malgo-1-x6dhm_1 |\u001b[0m SM_HP_EVAL_METRIC=mlogloss\n", "\u001b[36malgo-1-x6dhm_1 |\u001b[0m SM_MODULE_NAME=train_model\n", "\u001b[36malgo-1-x6dhm_1 |\u001b[0m SM_HP_BONITOO_PRICE_LIMIT=1000\n", "\u001b[36malgo-1-x6dhm_1 |\u001b[0m SM_HOSTS=[\"algo-1-x6dhm\"]\n", "\u001b[36malgo-1-x6dhm_1 |\u001b[0m SM_TRAINING_ENV={\"additional_framework_parameters\":{},\"channel_input_dirs\":{\"training\":\"/opt/ml/input/data/training\"},\"current_host\":\"algo-1-x6dhm\",\"framework_module\":\"sagemaker_xgboost_container.training:main\",\"hosts\":[\"algo-1-x6dhm\"],\"hyperparameters\":{\"bonitoo_price_limit\":1000,\"eta\":0.5,\"eval_metric\":\"mlogloss\",\"max_depth\":15,\"num_class\":8,\"num_round\":15,\"objective\":\"multi:softmax\"},\"input_config_dir\":\"/opt/ml/input/config\",\"input_data_config\":{\"training\":{\"TrainingInputMode\":\"File\"}},\"input_dir\":\"/opt/ml/input\",\"is_master\":true,\"job_name\":\"sagemaker-xgboost-2019-10-05-20-16-58-398\",\"log_level\":20,\"master_hostname\":\"algo-1-x6dhm\",\"model_dir\":\"/opt/ml/model\",\"module_dir\":\"s3://sagemaker-eu-central-1-029917565482/sagemaker-xgboost-2019-10-05-20-16-58-398/source/sourcedir.tar.gz\",\"module_name\":\"train_model\",\"network_interface_name\":\"eth0\",\"num_cpus\":6,\"num_gpus\":0,\"output_data_dir\":\"/opt/ml/output/data\",\"output_dir\":\"/opt/ml/output\",\"output_intermediate_dir\":\"/opt/ml/output/intermediate\",\"resource_config\":{\"current_host\":\"algo-1-x6dhm\",\"hosts\":[\"algo-1-x6dhm\"]},\"user_entry_point\":\"train_model.py\"}\n", "\u001b[36malgo-1-x6dhm_1 |\u001b[0m SM_MODEL_DIR=/opt/ml/model\n", "\u001b[36malgo-1-x6dhm_1 |\u001b[0m \n", "\u001b[36malgo-1-x6dhm_1 |\u001b[0m Invoking script with the following command:\n", "\u001b[36malgo-1-x6dhm_1 |\u001b[0m \n", "\u001b[36malgo-1-x6dhm_1 |\u001b[0m /usr/bin/python3 -m train_model --bonitoo_price_limit 1000 --eta 0.5 --eval_metric mlogloss --max_depth 15 --num_class 8 --num_round 15 --objective multi:softmax\n", "\u001b[36malgo-1-x6dhm_1 |\u001b[0m \n", "\u001b[36malgo-1-x6dhm_1 |\u001b[0m \n" ] }, { "name": "stdout", "output_type": "stream", "text": [ "\u001b[36malgo-1-x6dhm_1 |\u001b[0m ERROR:sagemaker-containers:ExecuteUserScriptError:\n", "\u001b[36malgo-1-x6dhm_1 |\u001b[0m Command \"/usr/bin/python3 -m train_model --bonitoo_price_limit 1000 --eta 0.5 --eval_metric mlogloss --max_depth 15 --num_class 8 --num_round 15 --objective multi:softmax\"\n", "\u001b[36malgo-1-x6dhm_1 |\u001b[0m INFO:root:hyperparameters {'num_round': 15, 'num_class': 8, 'objective': 'multi:softmax', 'eta': 0.5, 'max_depth': 15, 'eval_metric': ['mlogloss']}\n", "\u001b[36malgo-1-x6dhm_1 |\u001b[0m INFO:root:channels {'training': {'TrainingInputMode': 'File'}}\n", "\u001b[36malgo-1-x6dhm_1 |\u001b[0m INFO:root:Determined delimiter of CSV input is ','\n", "\u001b[36malgo-1-x6dhm_1 |\u001b[0m INFO:root:Loading csv file export.csv\n", "\u001b[36malgo-1-x6dhm_1 |\u001b[0m INFO:root:Preprocessing start\n", "\u001b[36malgo-1-x6dhm_1 |\u001b[0m /usr/local/lib/python3.5/dist-packages/pandas/core/indexing.py:543: SettingWithCopyWarning: \n", "\u001b[36malgo-1-x6dhm_1 |\u001b[0m A value is trying to be set on a copy of a slice from a DataFrame.\n", "\u001b[36malgo-1-x6dhm_1 |\u001b[0m Try using .loc[row_indexer,col_indexer] = value instead\n", "\u001b[36malgo-1-x6dhm_1 |\u001b[0m \n", "\u001b[36malgo-1-x6dhm_1 |\u001b[0m See the caveats in the documentation: http://pandas.pydata.org/pandas-docs/stable/indexing.html#indexing-view-versus-copy\n", "\u001b[36malgo-1-x6dhm_1 |\u001b[0m self.obj[item] = s\n", "\u001b[36malgo-1-x6dhm_1 |\u001b[0m /usr/local/lib/python3.5/dist-packages/pandas/core/indexing.py:362: SettingWithCopyWarning: \n", "\u001b[36malgo-1-x6dhm_1 |\u001b[0m A value is trying to be set on a copy of a slice from a DataFrame.\n", "\u001b[36malgo-1-x6dhm_1 |\u001b[0m Try using .loc[row_indexer,col_indexer] = value instead\n", "\u001b[36malgo-1-x6dhm_1 |\u001b[0m \n", "\u001b[36malgo-1-x6dhm_1 |\u001b[0m See the caveats in the documentation: http://pandas.pydata.org/pandas-docs/stable/indexing.html#indexing-view-versus-copy\n", "\u001b[36malgo-1-x6dhm_1 |\u001b[0m self.obj[key] = _infer_fill_value(value)\n", "\u001b[36malgo-1-x6dhm_1 |\u001b[0m INFO:root:Computing cached times\n", "\u001b[36malgo-1-x6dhm_1 |\u001b[0m INFO:root:Splitting dataset with ration 0.800000\n", "\u001b[36malgo-1-x6dhm_1 |\u001b[0m /usr/local/lib/python3.5/dist-packages/xgboost/core.py:587: FutureWarning: Series.base is deprecated and will be removed in a future version\n", "\u001b[36malgo-1-x6dhm_1 |\u001b[0m if getattr(data, 'base', None) is not None and \\\n", "\u001b[36malgo-1-x6dhm_1 |\u001b[0m /usr/local/lib/python3.5/dist-packages/xgboost/core.py:588: FutureWarning: Series.base is deprecated and will be removed in a future version\n", "\u001b[36malgo-1-x6dhm_1 |\u001b[0m data.base is not None and isinstance(data, np.ndarray) \\\n", "\u001b[36malgo-1-x6dhm_1 |\u001b[0m INFO:root:Single node training.\n", "\u001b[36malgo-1-x6dhm_1 |\u001b[0m INFO:root:Train matrix has 25393 rows\n", "\u001b[36malgo-1-x6dhm_1 |\u001b[0m INFO:root:Validation matrix has 6314 rows\n", "\u001b[36malgo-1-x6dhm_1 |\u001b[0m INFO:root:cols: ['index', 'adults', 'children', 'infants', 'input.price', 'input.tax', 'status', 'output.price', 'output.tax', 'duration', 'client.channel_codes', 'type_codes', 'flight.inboundSegments.departure_codes', 'flight.inboundSegments.arrival_codes', 'flight.inboundSegments.origin.airportCode_codes', 'flight.inboundSegments.destination.airportCode_codes', 'flight.inboundSegments.flightNumber_codes', 'flight.inboundSegments.travelClass_codes', 'flight.inboundSegments.bookingCode_codes', 'flight.inboundSegments.availability_codes', 'flight.inboundSegments.elapsedFlyingTime_codes', 'flight.outboundSegments.departure_codes', 'flight.outboundSegments.arrival_codes', 'flight.outboundSegments.origin.airportCode_codes', 'flight.outboundSegments.destination.airportCode_codes', 'flight.outboundSegments.flightNumber_codes', 'flight.outboundSegments.travelClass_codes', 'flight.outboundSegments.bookingCode_codes', 'flight.outboundSegments.availability_codes', 'flight.outboundSegments.elapsedFlyingTime_codes', 'flight.inboundEFT_codes', 'flight.outboundEFT_codes', 'input.currency_codes', 'output.currency_codes', 'oneWay_codes']\n", "\u001b[36malgo-1-x6dhm_1 |\u001b[0m Traceback (most recent call last):\n", "\u001b[36malgo-1-x6dhm_1 |\u001b[0m File \"/usr/lib/python3.5/runpy.py\", line 184, in _run_module_as_main\n", "\u001b[36malgo-1-x6dhm_1 |\u001b[0m \"__main__\", mod_spec)\n", "\u001b[36malgo-1-x6dhm_1 |\u001b[0m File \"/usr/lib/python3.5/runpy.py\", line 85, in _run_code\n", "\u001b[36malgo-1-x6dhm_1 |\u001b[0m exec(code, run_globals)\n", "\u001b[36malgo-1-x6dhm_1 |\u001b[0m File \"/opt/ml/code/train_model.py\", line 417, in \n", "\u001b[36malgo-1-x6dhm_1 |\u001b[0m checkpoint_config=checkpoint_config\n", "\u001b[36malgo-1-x6dhm_1 |\u001b[0m File \"/opt/ml/code/train_model.py\", line 320, in sagemaker_train\n", "\u001b[36malgo-1-x6dhm_1 |\u001b[0m train_job(**train_args)\n", "\u001b[36malgo-1-x6dhm_1 |\u001b[0m File \"/opt/ml/code/train_model.py\", line 364, in train_job\n", "\u001b[36malgo-1-x6dhm_1 |\u001b[0m raise Exception(\"cols: %s\" % str(train_dmatrix.feature_names))\n", "\u001b[36malgo-1-x6dhm_1 |\u001b[0m Exception: cols: ['index', 'adults', 'children', 'infants', 'input.price', 'input.tax', 'status', 'output.price', 'output.tax', 'duration', 'client.channel_codes', 'type_codes', 'flight.inboundSegments.departure_codes', 'flight.inboundSegments.arrival_codes', 'flight.inboundSegments.origin.airportCode_codes', 'flight.inboundSegments.destination.airportCode_codes', 'flight.inboundSegments.flightNumber_codes', 'flight.inboundSegments.travelClass_codes', 'flight.inboundSegments.bookingCode_codes', 'flight.inboundSegments.availability_codes', 'flight.inboundSegments.elapsedFlyingTime_codes', 'flight.outboundSegments.departure_codes', 'flight.outboundSegments.arrival_codes', 'flight.outboundSegments.origin.airportCode_codes', 'flight.outboundSegments.destination.airportCode_codes', 'flight.outboundSegments.flightNumber_codes', 'flight.outboundSegments.travelClass_codes', 'flight.outboundSegments.bookingCode_codes', 'flight.outboundSegments.availability_codes', 'flight.outboundSegments.elapsedFlyingTime_codes', 'flight.inboundEFT_codes', 'flight.outboundEFT_codes', 'input.currency_codes', 'output.currency_codes', 'oneWay_codes']\n", "\u001b[36mtmptao5hpuc_algo-1-x6dhm_1 exited with code 1\n", "\u001b[0mAborting on container exit...\n" ] }, { "ename": "RuntimeError", "evalue": "Failed to run: ['docker-compose', '-f', '/tmp/tmptao5hpuc/docker-compose.yaml', 'up', '--build', '--abort-on-container-exit'], Process exited with code: 1", "output_type": "error", "traceback": [ "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m", "\u001b[0;31mRuntimeError\u001b[0m Traceback (most recent call last)", "\u001b[0;32m~/aprar/bonitoo/.venv/lib/python3.7/site-packages/sagemaker/local/image.py\u001b[0m in \u001b[0;36mtrain\u001b[0;34m(self, input_data_config, output_data_config, hyperparameters, job_name)\u001b[0m\n\u001b[1;32m 147\u001b[0m \u001b[0;32mtry\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 148\u001b[0;31m \u001b[0m_stream_output\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mprocess\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 149\u001b[0m \u001b[0;32mexcept\u001b[0m \u001b[0mRuntimeError\u001b[0m \u001b[0;32mas\u001b[0m \u001b[0me\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n", "\u001b[0;32m~/aprar/bonitoo/.venv/lib/python3.7/site-packages/sagemaker/local/image.py\u001b[0m in \u001b[0;36m_stream_output\u001b[0;34m(process)\u001b[0m\n\u001b[1;32m 656\u001b[0m \u001b[0;32mif\u001b[0m \u001b[0mexit_code\u001b[0m \u001b[0;34m!=\u001b[0m \u001b[0;36m0\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 657\u001b[0;31m \u001b[0;32mraise\u001b[0m \u001b[0mRuntimeError\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m\"Process exited with code: %s\"\u001b[0m \u001b[0;34m%\u001b[0m \u001b[0mexit_code\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 658\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n", "\u001b[0;31mRuntimeError\u001b[0m: Process exited with code: 1", "\nDuring handling of the above exception, another exception occurred:\n", "\u001b[0;31mRuntimeError\u001b[0m Traceback (most recent call last)", "\u001b[0;32m\u001b[0m in \u001b[0;36m\u001b[0;34m\u001b[0m\n\u001b[0;32m----> 1\u001b[0;31m \u001b[0mestimator\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mtf\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mfit\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m{\u001b[0m\u001b[0;34m'training'\u001b[0m\u001b[0;34m:\u001b[0m \u001b[0mtrain_input\u001b[0m\u001b[0;34m}\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 2\u001b[0m \u001b[0;31m#estimator = sklearn.attach('sagemaker-scikit-learn-2019-01-25-16-34-38-829')\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n", "\u001b[0;32m~/aprar/bonitoo/.venv/lib/python3.7/site-packages/sagemaker/estimator.py\u001b[0m in \u001b[0;36mfit\u001b[0;34m(self, inputs, wait, logs, job_name)\u001b[0m\n\u001b[1;32m 337\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_prepare_for_training\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mjob_name\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mjob_name\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 338\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 339\u001b[0;31m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mlatest_training_job\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0m_TrainingJob\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mstart_new\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0minputs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 340\u001b[0m \u001b[0;32mif\u001b[0m \u001b[0mwait\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 341\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mlatest_training_job\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mwait\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mlogs\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mlogs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n", "\u001b[0;32m~/aprar/bonitoo/.venv/lib/python3.7/site-packages/sagemaker/estimator.py\u001b[0m in \u001b[0;36mstart_new\u001b[0;34m(cls, estimator, inputs)\u001b[0m\n\u001b[1;32m 861\u001b[0m \u001b[0mcls\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_add_spot_checkpoint_args\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mlocal_mode\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mestimator\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mtrain_args\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 862\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 863\u001b[0;31m \u001b[0mestimator\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0msagemaker_session\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mtrain\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m**\u001b[0m\u001b[0mtrain_args\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 864\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 865\u001b[0m \u001b[0;32mreturn\u001b[0m \u001b[0mcls\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mestimator\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0msagemaker_session\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mestimator\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_current_job_name\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n", "\u001b[0;32m~/aprar/bonitoo/.venv/lib/python3.7/site-packages/sagemaker/session.py\u001b[0m in \u001b[0;36mtrain\u001b[0;34m(self, input_mode, input_config, role, job_name, output_config, resource_config, vpc_config, hyperparameters, stop_condition, tags, metric_definitions, enable_network_isolation, image, algorithm_arn, encrypt_inter_container_traffic, train_use_spot_instances, checkpoint_s3_uri, checkpoint_local_path)\u001b[0m\n\u001b[1;32m 390\u001b[0m \u001b[0mLOGGER\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0minfo\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m\"Creating training-job with name: %s\"\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mjob_name\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 391\u001b[0m \u001b[0mLOGGER\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mdebug\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m\"train request: %s\"\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mjson\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mdumps\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mtrain_request\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mindent\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;36m4\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 392\u001b[0;31m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0msagemaker_client\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mcreate_training_job\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m**\u001b[0m\u001b[0mtrain_request\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 393\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 394\u001b[0m def compile_model(\n", "\u001b[0;32m~/aprar/bonitoo/.venv/lib/python3.7/site-packages/sagemaker/local/local_session.py\u001b[0m in \u001b[0;36mcreate_training_job\u001b[0;34m(self, TrainingJobName, AlgorithmSpecification, OutputDataConfig, ResourceConfig, InputDataConfig, **kwargs)\u001b[0m\n\u001b[1;32m 99\u001b[0m \u001b[0mtraining_job\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0m_LocalTrainingJob\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mcontainer\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 100\u001b[0m \u001b[0mhyperparameters\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mkwargs\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;34m\"HyperParameters\"\u001b[0m\u001b[0;34m]\u001b[0m \u001b[0;32mif\u001b[0m \u001b[0;34m\"HyperParameters\"\u001b[0m \u001b[0;32min\u001b[0m \u001b[0mkwargs\u001b[0m \u001b[0;32melse\u001b[0m \u001b[0;34m{\u001b[0m\u001b[0;34m}\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 101\u001b[0;31m \u001b[0mtraining_job\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mstart\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mInputDataConfig\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mOutputDataConfig\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mhyperparameters\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mTrainingJobName\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 102\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 103\u001b[0m \u001b[0mLocalSagemakerClient\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_training_jobs\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0mTrainingJobName\u001b[0m\u001b[0;34m]\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mtraining_job\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n", "\u001b[0;32m~/aprar/bonitoo/.venv/lib/python3.7/site-packages/sagemaker/local/entities.py\u001b[0m in \u001b[0;36mstart\u001b[0;34m(self, input_data_config, output_data_config, hyperparameters, job_name)\u001b[0m\n\u001b[1;32m 87\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 88\u001b[0m self.model_artifacts = self.container.train(\n\u001b[0;32m---> 89\u001b[0;31m \u001b[0minput_data_config\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0moutput_data_config\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mhyperparameters\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mjob_name\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 90\u001b[0m )\n\u001b[1;32m 91\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mend_time\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mdatetime\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mdatetime\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mnow\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n", "\u001b[0;32m~/aprar/bonitoo/.venv/lib/python3.7/site-packages/sagemaker/local/image.py\u001b[0m in \u001b[0;36mtrain\u001b[0;34m(self, input_data_config, output_data_config, hyperparameters, job_name)\u001b[0m\n\u001b[1;32m 151\u001b[0m \u001b[0;31m# which contains the exit code and append the command line to it.\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 152\u001b[0m \u001b[0mmsg\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;34m\"Failed to run: %s, %s\"\u001b[0m \u001b[0;34m%\u001b[0m \u001b[0;34m(\u001b[0m\u001b[0mcompose_command\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mstr\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0me\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 153\u001b[0;31m \u001b[0;32mraise\u001b[0m \u001b[0mRuntimeError\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mmsg\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 154\u001b[0m \u001b[0;32mfinally\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 155\u001b[0m \u001b[0martifacts\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mretrieve_artifacts\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mcompose_data\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0moutput_data_config\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mjob_name\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n", "\u001b[0;31mRuntimeError\u001b[0m: Failed to run: ['docker-compose', '-f', '/tmp/tmptao5hpuc/docker-compose.yaml', 'up', '--build', '--abort-on-container-exit'], Process exited with code: 1" ] } ], "source": [ "estimator = tf.fit({'training': train_input})\n", "#estimator = sklearn.attach('sagemaker-scikit-learn-2019-01-25-16-34-38-829')" ] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": [] } ], "metadata": { "kernelspec": { "display_name": "Python 3", "language": "python", "name": "python3" }, "language_info": { "codemirror_mode": { "name": "ipython", "version": 3 }, "file_extension": ".py", "mimetype": "text/x-python", "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", "version": "3.7.3" } }, "nbformat": 4, "nbformat_minor": 2 }