Interruption in Detecting and Categorizing Brick Kilns from Satellite Imagery

Anonymous User · ‎11-19-2020

Can anyone please help me with this, I was not able to train the model to detect the brick klins

Traceback (most recent call last):
  File "/opt/conda/lib/python3.6/multiprocessing/queues.py", line 234, in _feed
    obj = _ForkingPickler.dumps(obj)
Traceback (most recent call last):
  File "/opt/conda/lib/python3.6/multiprocessing/reduction.py", line 51, in dumps
    cls(buf, protocol).dump(obj)
  File "/opt/conda/lib/python3.6/site-packages/torch/multiprocessing/reductions.py", line 333, in reduce_storage
    fd, size = storage._share_fd_()
  File "/opt/conda/lib/python3.6/multiprocessing/queues.py", line 234, in _feed
    obj = _ForkingPickler.dumps(obj)
  File "/opt/conda/lib/python3.6/multiprocessing/reduction.py", line 51, in dumps
    cls(buf, protocol).dump(obj)
  File "/opt/conda/lib/python3.6/site-packages/torch/multiprocessing/reductions.py", line 333, in reduce_storage
    fd, size = storage._share_fd_()
RuntimeError: unable to write to file </torch_670_3343120479>
RuntimeError: unable to write to file </torch_669_2064567713>

---------------------------------------------------------------------------
RuntimeError                              Traceback (most recent call last)
/opt/conda/lib/python3.6/site-packages/torch/utils/data/dataloader.py in _try_get_data(self, timeout)
    760         try:
--> 761             data = self._data_queue.get(timeout=timeout)
    762             return (True, data)

/opt/conda/lib/python3.6/multiprocessing/queues.py in get(self, block, timeout)
    103                     timeout = deadline - time.monotonic()
--> 104                     if not self._poll(timeout):
    105                         raise Empty

/opt/conda/lib/python3.6/multiprocessing/connection.py in poll(self, timeout)
    256         self._check_readable()
--> 257         return self._poll(timeout)
    258 

/opt/conda/lib/python3.6/multiprocessing/connection.py in _poll(self, timeout)
    413     def _poll(self, timeout):
--> 414         r = wait([self], timeout)
    415         return bool(r)

/opt/conda/lib/python3.6/multiprocessing/connection.py in wait(object_list, timeout)
    910             while True:
--> 911                 ready = selector.select(timeout)
    912                 if ready:

/opt/conda/lib/python3.6/selectors.py in select(self, timeout)
    375             try:
--> 376                 fd_event_list = self._poll.poll(timeout)
    377             except InterruptedError:

/opt/conda/lib/python3.6/site-packages/torch/utils/data/_utils/signal_handling.py in handler(signum, frame)
     65         # Python can still get and update the process status successfully.
---> 66         _error_if_any_worker_fails()
     67         if previous_handler is not None:

RuntimeError: DataLoader worker (pid 668) is killed by signal: Bus error. It is possible that dataloader's workers are out of shared memory. Please try to raise your shared memory limit.

During handling of the above exception, another exception occurred:

RuntimeError                              Traceback (most recent call last)
<ipython-input-24-9197f7b90f5b> in <module>
----> 1 model.fit(epochs=10, lr=0.001)

/opt/conda/lib/python3.6/site-packages/arcgis/learn/models/_arcgis_model.py in fit(self, epochs, lr, one_cycle, early_stopping, checkpoint, tensorboard, **kwargs)
    543 
    544         if one_cycle:
--> 545             self.learn.fit_one_cycle(epochs, lr, callbacks=callbacks, **kwargs)
    546         else:
    547             self.learn.fit(epochs, lr, callbacks=callbacks, **kwargs)

/opt/conda/lib/python3.6/site-packages/fastai/train.py in fit_one_cycle(learn, cyc_len, max_lr, moms, div_factor, pct_start, final_div, wd, callbacks, tot_epochs, start_epoch)
     21     callbacks.append(OneCycleScheduler(learn, max_lr, moms=moms, div_factor=div_factor, pct_start=pct_start,
     22                                        final_div=final_div, tot_epochs=tot_epochs, start_epoch=start_epoch))
---> 23     learn.fit(cyc_len, max_lr, wd=wd, callbacks=callbacks)
     24 
     25 def fit_fc(learn:Learner, tot_epochs:int=1, lr:float=defaults.lr,  moms:Tuple[float,float]=(0.95,0.85), start_pct:float=0.72,

/opt/conda/lib/python3.6/site-packages/fastai/basic_train.py in fit(self, epochs, lr, wd, callbacks)
    198         else: self.opt.lr,self.opt.wd = lr,wd
    199         callbacks = [cb(self) for cb in self.callback_fns + listify(defaults.extra_callback_fns)] + listify(callbacks)
--> 200         fit(epochs, self, metrics=self.metrics, callbacks=self.callbacks+callbacks)
    201 
    202     def create_opt(self, lr:Floats, wd:Floats=0.)->None:

/opt/conda/lib/python3.6/site-packages/fastai/basic_train.py in fit(epochs, learn, callbacks, metrics)
     97             cb_handler.set_dl(learn.data.train_dl)
     98             cb_handler.on_epoch_begin()
---> 99             for xb,yb in progress_bar(learn.data.train_dl, parent=pbar):
    100                 xb, yb = cb_handler.on_batch_begin(xb, yb)
    101                 loss = loss_batch(learn.model, xb, yb, learn.loss_func, learn.opt, cb_handler)

/opt/conda/lib/python3.6/site-packages/fastprogress/fastprogress.py in __iter__(self)
     45         except Exception as e:
     46             self.on_interrupt()
---> 47             raise e
     48 
     49     def update(self, val):

/opt/conda/lib/python3.6/site-packages/fastprogress/fastprogress.py in __iter__(self)
     39         if self.total != 0: self.update(0)
     40         try:
---> 41             for i,o in enumerate(self.gen):
     42                 if i >= self.total: break
     43                 yield o

/opt/conda/lib/python3.6/site-packages/fastai/basic_data.py in __iter__(self)
     73     def __iter__(self):
     74         "Process and returns items from `DataLoader`."
---> 75         for b in self.dl: yield self.proc_batch(b)
     76 
     77     @classmethod

/opt/conda/lib/python3.6/site-packages/torch/utils/data/dataloader.py in __next__(self)
    343 
    344     def __next__(self):
--> 345         data = self._next_data()
    346         self._num_yielded += 1
    347         if self._dataset_kind == _DatasetKind.Iterable and \

/opt/conda/lib/python3.6/site-packages/torch/utils/data/dataloader.py in _next_data(self)
    839 
    840             assert not self._shutdown and self._tasks_outstanding > 0
--> 841             idx, data = self._get_data()
    842             self._tasks_outstanding -= 1
    843 

/opt/conda/lib/python3.6/site-packages/torch/utils/data/dataloader.py in _get_data(self)
    806         else:
    807             while True:
--> 808                 success, data = self._try_get_data()
    809                 if success:
    810                     return data

/opt/conda/lib/python3.6/site-packages/torch/utils/data/dataloader.py in _try_get_data(self, timeout)
    772             if len(failed_workers) > 0:
    773                 pids_str = ', '.join(str(w.pid) for w in failed_workers)
--> 774                 raise RuntimeError('DataLoader worker (pid(s) {}) exited unexpectedly'.format(pids_str))
    775             if isinstance(e, queue.Empty):
    776                 return (False, None)

RuntimeError: DataLoader worker (pid(s) 668) exited unexpectedly