1 Answer
- Newest
- Most votes
- Most comments
0
Scaled up a bigger master and and core instance group size to get that bit working! Still get errors on different things that id except to work.
example:
The AWS spark example code:
df = spark.read.parquet('s3://amazon-reviews-pds/parquet/product_category=Books/*.parquet')
I get error:
---------------------------------------------------------------------------
KeyboardInterrupt Traceback (most recent call last)
/mnt/notebook-env/lib/python3.7/site-packages/sparkmagic/livyclientlib/command.py in execute(self, session)
60 statement_id = response["id"]
---> 61 output = self._get_statement_output(session, statement_id)
62 except KeyboardInterrupt as e:
/mnt/notebook-env/lib/python3.7/site-packages/sparkmagic/livyclientlib/command.py in _get_statement_output(self, session, statement_id)
133 progress.value = statement.get("progress", 0.0)
--> 134 session.sleep(retries)
135 retries += 1
/mnt/notebook-env/lib/python3.7/site-packages/sparkmagic/livyclientlib/livysession.py in sleep(self, retries)
362 def sleep(self, retries):
--> 363 sleep(self._policy.seconds_to_sleep(retries))
364
KeyboardInterrupt:
During handling of the above exception, another exception occurred:
LivyClientTimeoutException Traceback (most recent call last)
/mnt/notebook-env/lib/python3.7/site-packages/sparkmagic/livyclientlib/command.py in execute(self, session)
77 )
---> 78 session.wait_for_idle()
79 except:
/mnt/notebook-env/lib/python3.7/site-packages/sparkmagic/livyclientlib/livysession.py in wait_for_idle(self, seconds_to_wait)
336 self.logger.error(error)
--> 337 raise LivyClientTimeoutException(error)
338
LivyClientTimeoutException: Session 0 did not reach idle status in time. Current status is busy.
During handling of the above exception, another exception occurred:
SparkStatementCancellationFailedException Traceback (most recent call last)
<ipython-input-9-8c1806c06a24> in <module>
----> 1 get_ipython().run_cell_magic('spark', '', "df = spark.read.parquet('s3://amazon-reviews-pds/parquet/product_category=Books/*.parquet')\n")
/mnt/notebook-env/lib/python3.7/site-packages/IPython/core/interactiveshell.py in run_cell_magic(self, magic_name, line, cell)
2417 with self.builtin_trap:
2418 args = (magic_arg_s, cell)
-> 2419 result = fn(*args, **kwargs)
2420 return result
2421
/mnt/notebook-env/lib/python3.7/site-packages/decorator.py in fun(*args, **kw)
230 if not kwsyntax:
231 args, kw = fix(args, kw, sig)
--> 232 return caller(func, *(extras + args), **kw)
233 fun.__name__ = func.__name__
234 fun.__doc__ = func.__doc__
/mnt/notebook-env/lib/python3.7/site-packages/IPython/core/magic.py in <lambda>(f, *a, **k)
185 # but it's overkill for just that one bit of state.
186 def magic_deco(arg):
--> 187 call = lambda f, *a, **k: f(*a, **k)
188
189 if callable(arg):
/mnt/notebook-env/lib/python3.7/site-packages/sparkmagic/livyclientlib/exceptions.py in wrapped(self, *args, **kwargs)
163 def wrapped(self, *args, **kwargs):
164 try:
--> 165 out = f(self, *args, **kwargs)
166 except Exception as err:
167 if conf.all_errors_are_fatal():
/mnt/notebook-env/lib/python3.7/site-packages/sparkmagic/livyclientlib/exceptions.py in wrapped(self, *args, **kwargs)
124 def wrapped(self, *args, **kwargs):
125 try:
--> 126 out = f(self, *args, **kwargs)
127 except exceptions_to_handle as err:
128 if conf.all_errors_are_fatal():
/mnt/notebook-env/lib/python3.7/site-packages/sparkmagic/kernels/kernelmagics.py in spark(self, line, cell, local_ns)
483 args.samplefraction,
484 None,
--> 485 coerce,
486 )
487
/mnt/notebook-env/lib/python3.7/site-packages/sparkmagic/magics/sparkmagicsbase.py in execute_spark(self, cell, output_var, samplemethod, maxrows, samplefraction, session_name, coerce, output_handler, cell_kind)
128
129 (success, out, mimetype) = self.spark_controller.run_command(
--> 130 Command(cell, None, cell_kind), session_name
131 )
132 if not success:
/mnt/notebook-env/lib/python3.7/site-packages/sparkmagic/livyclientlib/sparkcontroller.py in run_command(self, command, client_name)
39 def run_command(self, command, client_name=None):
40 session_to_use = self.get_session_by_name_or_default(client_name)
---> 41 return command.execute(session_to_use)
42
43 def run_sqlquery(self, sqlquery, client_name=None):
/mnt/notebook-env/lib/python3.7/site-packages/sparkmagic/livyclientlib/command.py in execute(self, session)
79 except:
80 raise SparkStatementCancellationFailedException(
---> 81 COMMAND_CANCELLATION_FAILED_MSG
82 )
83 else:
SparkStatementCancellationFailedException: Interrupted by user but Livy failed to cancel the Spark statement. The Livy session might have become unusable.
But i can run all the steps of the AWS example word-count-woth-pyspark...
Running boto3 never seems to end either:
example :
s3 = boto3.client('s3')
response = s3.list_buckets()
# Output the bucket names
print('Existing buckets:')
for bucket in response['Buckets']:
print(f' {bucket["Name"]}')
I get an orange field in notebook and no response ever comes. soon as i press run it gives me a Last executed at
but thats it.. never finishes
answered a year ago
Relevant content
- asked 2 years ago
- asked 5 months ago
- AWS OFFICIALUpdated 2 years ago
- AWS OFFICIALUpdated a year ago
- AWS OFFICIALUpdated a year ago
- AWS OFFICIALUpdated a year ago