I am trying to deploy my RandomForestClassifier on Amazon Sagemaker using Python SDK. I have been following this example https://github.com/aws/amazon-sagemaker-examples/blob/main/sagemaker-script-mode/sagemaker-script-mode.ipynb but keep getting an error that the train file was not found. I think the file were not uploaded to the correct channel. When I run the script as follows it works fine.
! python script_rf.py --model-dir ./ \
--train ./ \
--test ./ \
This is my script code:
# inference functions ---------------
def model_fn(model_dir):
clf = joblib.load(os.path.join(model_dir, "model.joblib"))
return clf
if __name__ =='__main__':
print('extracting arguments')
parser = argparse.ArgumentParser()
# hyperparameters sent by the client are passed as command-line arguments to the script.
parser.add_argument('--max_depth', type=int, default=2)
parser.add_argument('--n_estimators', type=int, default=100)
parser.add_argument('--random_state', type=int, default=0)
# Data, model, and output directories
parser.add_argument('--model-dir', type=str, default=os.environ.get('SM_MODEL_DIR'))
parser.add_argument('--train', type=str, default=os.environ.get('SM_CHANNEL_TEST'))
parser.add_argument('--test', type=str, default=os.environ.get('SM_CHANNEL_TEST'))
parser.add_argument('--train-file', type=str, default='revenue_train.csv')
parser.add_argument('--test-file', type=str, default='revenue_test.csv')
args, _ = parser.parse_known_args()
print('reading data')
train_df = pd.read_csv(os.path.join(args.train, args.train_file))
test_df = pd.read_csv(os.path.join(args.test, args.test_file))
if len(train_df) == 0:
raise ValueError(('There are no files in {}.\n').format(args.train, "train"))
print('building training and testing datasets')
attributes = ['available_minutes_100','ampido_slots_amount','ampido_slots_amount_100','ampido_slots_amount_200','ampido_slots_amount_300','min_dist_loc','count_event','min_dist_phouses','count_phouses','min_dist_stops','count_stops','min_dist_tickets','count_tickets','min_dist_google','min_dist_psa','count_psa']
X_train = train_df[attributes]
X_test = test_df[attributes]
y_train = train_df['target']
y_test = test_df['target']
# train
print('training model')
model = RandomForestClassifier(
max_depth =args.max_depth, n_estimators = args.n_estimators)
model.fit(X_train, y_train)
# persist model
path = os.path.join(args.model_dir, "model_rf.joblib")
joblib.dump(model, path)
print('model persisted at ' + path)
# print accuracy and confusion matrix
print('validating model')
y_pred=model.predict(X_test)
print('Confusion Matrix:')
result = confusion_matrix(y_test, y_pred)
print(result)
print('Accuracy:')
result2 = accuracy_score(y_test, y_pred)
print(result2)
the error is raised in the train_df line of the script (FileNotFoundError: [Errno 2] No such file or directory: '/opt/ml/input/data/test/revenue_train.csv').
I tried specifying the input parameters:
# change channel input dirs
inputs = {
"train": "ampido-exports/production/revenue_train",
"test": "ampido-exports/production/revenue_test",
}
from sagemaker.sklearn.estimator import SKLearn
enable_local_mode_training = False
hyperparameters = {"max_depth": 2, 'random_state':0, "n_estimators": 100}
if enable_local_mode_training:
train_instance_type = "local"
inputs = {"train": trainpath, "test": testpath}
else:
train_instance_type = "ml.c5.xlarge"
inputs = {"train": trainpath, "test": testpath}
estimator_parameters = {
"entry_point": "script_rf.py",
"framework_version": "1.0-1",
"py_version": "py3",
"instance_type": train_instance_type,
"instance_count": 1,
"hyperparameters": hyperparameters,
"role": role,
"base_job_name": "randomforestclassifier-model",
'channel_input_dirs' : inputs
}
estimator = SKLearn(**estimator_parameters)
estimator.fit(inputs)
but i still get the error FileNotFoundError: [Errno 2] No such file or directory: '/opt/ml/input/data/test/revenue_train.csv