Sourcery refactored master branch

fastai/adaptive_softmax.py

@@ -55,7 +55,9 @@ class AdaptiveLoss(nn.Module):
     def __init__(self, cutoff):
         super().__init__()
         self.cutoff = cutoff
-        self.criterions = nn.ModuleList([nn.CrossEntropyLoss(size_average=False) for i in self.cutoff])
+        self.criterions = nn.ModuleList(
+            [nn.CrossEntropyLoss(size_average=False) for _ in self.cutoff]
+        )
 
     def remap_target(self, target):
         new_target = [target.clone()]

fastai/column_data.py

@@ -147,10 +147,7 @@ def forward(self, x_cat, x_cont):
             x = d(x)
         x = self.outp(x)
         if not self.is_reg:
-            if self.is_multi:
-                x = F.sigmoid(x)
-            else:
-                x = F.log_softmax(x)
+            x = F.sigmoid(x) if self.is_multi else F.log_softmax(x)
         elif self.y_range:
             x = F.sigmoid(x)
             x = x*(self.y_range[1] - self.y_range[0])

fastai/conv_learner.py

@@ -34,8 +34,7 @@ def __init__(self, f, c, is_multi, is_reg, ps=None, xtra_fc=None, xtra_cut=0, cu
         if ps is None: ps = [0.25]*len(xtra_fc) + [0.5]
         self.ps,self.xtra_fc = ps,xtra_fc
 
-        if f in model_meta: cut,self.lr_cut = model_meta[f]
-        else: cut,self.lr_cut = 0,0
+        cut,self.lr_cut = model_meta[f] if f in model_meta else (0, 0)
         cut-=xtra_cut
         layers = cut_model(f(pretrained), cut)
         self.nf = model_features[f] if f in model_features else (num_features(layers)*2)
@@ -45,8 +44,7 @@ def __init__(self, f, c, is_multi, is_reg, ps=None, xtra_fc=None, xtra_cut=0, cu
         n_fc = len(self.xtra_fc)+1
         if not isinstance(self.ps, list): self.ps = [self.ps]*n_fc
 
-        if custom_head: fc_layers = [custom_head]
-        else: fc_layers = self.get_fc_layers()
+        fc_layers = [custom_head] if custom_head else self.get_fc_layers()
         self.n_fc = len(fc_layers)
         self.fc_model = to_gpu(nn.Sequential(*fc_layers))
         if not custom_head: apply_init(self.fc_model, kaiming_normal)
@@ -180,8 +178,7 @@ def save_fc1(self):
         if len(self.activations[1])!=len(self.data.val_ds):
             predict_to_bcolz(m, self.data.val_dl, val_act)
         if self.data.test_dl and (len(self.activations[2])!=len(self.data.test_ds)):
-            if self.data.test_dl: predict_to_bcolz(m, self.data.test_dl, test_act)
-
+            predict_to_bcolz(m, self.data.test_dl, test_act)
         self.fc_data = ImageClassifierData.from_arrays(self.data.path,
                 (act, self.data.trn_y), (val_act, self.data.val_y), self.data.bs, classes=self.data.classes,
                 test = test_act if self.data.test_dl else None, num_workers=8)

fastai/core.py

@@ -15,8 +15,7 @@ def listify(x, y):
     return x
 
 def datafy(x):
-    if is_listy(x): return [o.data for o in x]
-    else:           return x.data
+    return [o.data for o in x] if is_listy(x) else x.data
 
 conv_dict = {np.dtype('int8'): torch.LongTensor, np.dtype('int16'): torch.LongTensor,
     np.dtype('int32'): torch.LongTensor, np.dtype('int64'): torch.LongTensor,
@@ -186,7 +185,7 @@ def chunk_iter(iterable, chunk_size):
     while True:
         chunk = []
         try:
-            for _ in range(chunk_size): chunk.append(next(iterable))
+            chunk.extend(next(iterable) for _ in range(chunk_size))
             yield chunk
         except StopIteration:
             if chunk: yield chunk

fastai/dataloader.py

@@ -69,7 +69,7 @@ def np_collate(self, batch):
             return {key: self.np_collate([d[key] for d in batch]) for key in b}
         elif isinstance(b, collections.Sequence):
             return [self.np_collate(samples) for samples in zip(*batch)]
-        raise TypeError(("batch must contain numbers, dicts or lists; found {}".format(type(b))))
+        raise TypeError(f"batch must contain numbers, dicts or lists; found {type(b)}")
 
     def get_batch(self, indices):
         res = self.np_collate([self.dataset[i] for i in indices])

fastai/dataset.py

@@ -94,9 +94,11 @@ def read_dir(path, folder):
     if any(fnames):
         return [os.path.relpath(f,path) for f in fnames]
     elif any(directories):
-        raise FileNotFoundError("{} has subdirectories but contains no files. Is your directory structure is correct?".format(full_path))
+        raise FileNotFoundError(
+            f"{full_path} has subdirectories but contains no files. Is your directory structure is correct?"
+        )
     else:
-        raise FileNotFoundError("{} folder doesn't exist or is empty".format(full_path))
+        raise FileNotFoundError(f"{full_path} folder doesn't exist or is empty")
 
 def read_dirs(path, folder):
     '''
@@ -172,16 +174,24 @@ def csv_source(folder, csv_file, skip_header=True, suffix='', continuous=False,
     return dict_source(folder, fnames, csv_labels, suffix, continuous)
 
 def dict_source(folder, fnames, csv_labels, suffix='', continuous=False):
-    all_labels = sorted(list(set(p for o in csv_labels.values() for p in ([] if type(o) == float else o))))
+    all_labels = sorted(
+        list(
+            {
+                p
+                for o in csv_labels.values()
+                for p in ([] if type(o) == float else o)
+            }
+        )
+    )
     full_names = [os.path.join(folder,str(fn)+suffix) for fn in fnames]
     if continuous:
         label_arr = np.array([np.array(csv_labels[i]).astype(np.float32)
                 for i in fnames])
     else:
         label2idx = {v:k for k,v in enumerate(all_labels)}
         label_arr = nhot_labels(label2idx, csv_labels, fnames, len(all_labels))
-        is_single = np.all(label_arr.sum(axis=1)==1)
-        if is_single: label_arr = np.argmax(label_arr, axis=1)
+        if is_single := np.all(label_arr.sum(axis=1) == 1):
+            label_arr = np.argmax(label_arr, axis=1)
     return full_names, label_arr, all_labels
 
 class BaseDataset(Dataset):
@@ -263,19 +273,20 @@ def open_image(fn):
     """
     flags = cv2.IMREAD_UNCHANGED+cv2.IMREAD_ANYDEPTH+cv2.IMREAD_ANYCOLOR
     if not os.path.exists(fn) and not str(fn).startswith("http"):
-        raise OSError('No such file or directory: {}'.format(fn))
+        raise OSError(f'No such file or directory: {fn}')
     elif os.path.isdir(fn) and not str(fn).startswith("http"):
-        raise OSError('Is a directory: {}'.format(fn))
+        raise OSError(f'Is a directory: {fn}')
     elif isdicom(fn):
         slice = pydicom.read_file(fn)
-        if slice.PhotometricInterpretation.startswith('MONOCHROME'):
-            # Make a fake RGB image
-            im = np.stack([slice.pixel_array]*3,-1)
-            return im / ((1 << slice.BitsStored)-1)
-        else:
+        if not slice.PhotometricInterpretation.startswith('MONOCHROME'):
             # No support for RGB yet, as it involves various color spaces.
             # It shouldn't be too difficult to add though, if needed.
-            raise OSError('Unsupported DICOM image with PhotometricInterpretation=={}'.format(slice.PhotometricInterpretation))
+            raise OSError(
+                f'Unsupported DICOM image with PhotometricInterpretation=={slice.PhotometricInterpretation}'
+            )
+        # Make a fake RGB image
+        im = np.stack([slice.pixel_array]*3,-1)
+        return im / ((1 << slice.BitsStored)-1)
     else:
         #res = np.array(Image.open(fn), dtype=np.float32)/255
         #if len(res.shape)==2: res = np.repeat(res[...,None],3,2)
@@ -290,7 +301,7 @@ def open_image(fn):
             if im is None: raise OSError(f'File not recognized by opencv: {fn}')
             return cv2.cvtColor(im, cv2.COLOR_BGR2RGB)
         except Exception as e:
-            raise OSError('Error handling image at: {}'.format(fn)) from e
+            raise OSError(f'Error handling image at: {fn}') from e
 
 class FilesDataset(BaseDataset):
     def __init__(self, fnames, transform, path):
@@ -437,12 +448,10 @@ def resize(self, targ_sz, new_path='tmp', resume=True, fn=None):
         operation was aborted)
         fn (function): optional custom resizing function
         """
-        new_ds = []
         dls = [self.trn_dl,self.val_dl,self.fix_dl,self.aug_dl]
-        if self.test_dl: dls += [self.test_dl, self.test_aug_dl]
-        else: dls += [None,None]
+        dls += [self.test_dl, self.test_aug_dl] if self.test_dl else [None,None]
         t = tqdm_notebook(dls)
-        for dl in t: new_ds.append(self.resized(dl, targ_sz, new_path, resume, fn))
+        new_ds = [self.resized(dl, targ_sz, new_path, resume, fn) for dl in t]
         t.close()
         return self.__class__(new_ds[0].path, new_ds, self.bs, self.num_workers, self.classes)
 
@@ -458,11 +467,10 @@ def get_ds(fn, trn, val, tfms, test=None, **kwargs):
             if isinstance(test, tuple):
                 test_lbls = test[1]
                 test = test[0]
+            elif len(trn[1].shape) == 1:
+                test_lbls = np.zeros((len(test),1))
             else:
-                if len(trn[1].shape) == 1:
-                    test_lbls = np.zeros((len(test),1))
-                else:
-                    test_lbls = np.zeros((len(test),trn[1].shape[1]))
+                test_lbls = np.zeros((len(test),trn[1].shape[1]))
             res += [
                 fn(test, test_lbls, tfms[1], **kwargs), # test
                 fn(test, test_lbls, tfms[0], **kwargs)  # test_aug
@@ -510,7 +518,9 @@ def from_paths(cls, path, bs=64, tfms=(None,None), trn_name='train', val_name='v
         Returns:
             ImageClassifierData
         """
-        assert not(tfms[0] is None or tfms[1] is None), "please provide transformations for your train and validation sets"
+        assert (
+            tfms[0] is not None and tfms[1] is not None
+        ), "please provide transformations for your train and validation sets"
         trn,val = [folder_source(path, o) for o in (trn_name, val_name)]
         if test_name:
             test = folder_source(path, test_name) if test_with_labels else read_dir(path, test_name)
@@ -546,7 +556,9 @@ def from_csv(cls, path, folder, csv_fname, bs=64, tfms=(None,None),
         Returns:
             ImageClassifierData
         """
-        assert not (tfms[0] is None or tfms[1] is None), "please provide transformations for your train and validation sets"
+        assert (
+            tfms[0] is not None and tfms[1] is not None
+        ), "please provide transformations for your train and validation sets"
         assert not (os.path.isabs(folder)), "folder needs to be a relative path"
         fnames,y,classes = csv_source(folder, csv_fname, skip_header, suffix, continuous=continuous, cat_separator=cat_separator)
         return cls.from_names_and_array(path, fnames, y, classes, val_idxs, test_name,
@@ -571,7 +583,9 @@ def from_path_and_array(cls, path, folder, y, classes=None, val_idxs=None, test_
         Returns:
             ImageClassifierData
         """
-        assert not (tfms[0] is None or tfms[1] is None), "please provide transformations for your train and validation sets"
+        assert (
+            tfms[0] is not None and tfms[1] is not None
+        ), "please provide transformations for your train and validation sets"
         assert not (os.path.isabs(folder)), "folder needs to be a relative path"
         fnames = np.core.defchararray.add(f'{folder}/', sorted(os.listdir(f'{path}{folder}')))
         return cls.from_names_and_array(path, fnames, y, classes, val_idxs, test_name,

fastai/imports.py

@@ -40,10 +40,9 @@ def in_ipynb():
 def clear_tqdm():
     inst = getattr(tq.tqdm, '_instances', None)
     if not inst: return
-    try:
-        for i in range(len(inst)): inst.pop().close()
-    except Exception:
-        pass
+    with contextlib.suppress(Exception):
+        for _ in range(len(inst)):
+            inst.pop().close()
 
 if in_notebook():
     def tqdm(*args, **kwargs):

fastai/layer_optimizer.py

@@ -48,16 +48,18 @@ def set_wds(self, wds):
         self.wds=wds
 
     def set_mom(self,momentum):
-        if 'betas' in self.opt.param_groups[0]:
-            for pg in self.opt.param_groups: pg['betas'] = (momentum, pg['betas'][1])
-        else:
-            for pg in self.opt.param_groups: pg['momentum'] = momentum
+        for pg in self.opt.param_groups:
+            if 'betas' in self.opt.param_groups[0]:
+                pg['betas'] = (momentum, pg['betas'][1])
+            else:
+                pg['momentum'] = momentum
 
     def set_beta(self,beta):
-        if 'betas' in self.opt.param_groups[0]:
-            for pg in self.opt.param_groups: pg['betas'] = (pg['betas'][0],beta)
-        elif 'alpha' in self.opt.param_groups[0]:
-            for pg in self.opt.param_groups: pg['alpha'] = beta
+        for pg in self.opt.param_groups:
+            if 'betas' in self.opt.param_groups[0]:
+                pg['betas'] = (pg['betas'][0],beta)
+            elif 'alpha' in self.opt.param_groups[0]:
+                pg['alpha'] = beta
 
     def set_opt_fn(self, opt_fn):
         if type(self.opt) != type(opt_fn(self.opt_params())):

fastai/learner.py

@@ -95,15 +95,18 @@ def unfreeze_groups(self, groups):
 
     def unfreeze(self): self.freeze_to(0)
 
-    def get_model_path(self, name): return os.path.join(self.models_path,name)+'.h5'
+    def get_model_path(self, name):
+        return f'{os.path.join(self.models_path, name)}.h5'
 
     def save(self, name): 
         save_model(self.model, self.get_model_path(name))
-        if hasattr(self, 'swa_model'): save_model(self.swa_model, self.get_model_path(name)[:-3]+'-swa.h5')
+        if hasattr(self, 'swa_model'):
+            save_model(self.swa_model, f'{self.get_model_path(name)[:-3]}-swa.h5')
 
     def load(self, name): 
         load_model(self.model, self.get_model_path(name))
-        if hasattr(self, 'swa_model'): load_model(self.swa_model, self.get_model_path(name)[:-3]+'-swa.h5')
+        if hasattr(self, 'swa_model'):
+            load_model(self.swa_model, f'{self.get_model_path(name)[:-3]}-swa.h5')
 
     def set_data(self, data): self.data_ = data
 
@@ -385,7 +388,8 @@ def predict_array(self, arr):
         Returns:
             a numpy array containing the predictions from the model
         """
-        if not isinstance(arr, np.ndarray): raise OSError(f'Not valid numpy array')
+        if not isinstance(arr, np.ndarray):
+            raise OSError('Not valid numpy array')
         self.model.eval()
         return to_np(self.model(to_gpu(V(T(arr)))))
 
@@ -411,7 +415,10 @@ def TTA(self, n_aug=4, is_test=False):
         dl2 = self.data.test_aug_dl if is_test else self.data.aug_dl
         preds1,targs = predict_with_targs(self.model, dl1)
         preds1 = [preds1]*math.ceil(n_aug/4)
-        preds2 = [predict_with_targs(self.model, dl2)[0] for i in tqdm(range(n_aug), leave=False)]
+        preds2 = [
+            predict_with_targs(self.model, dl2)[0]
+            for _ in tqdm(range(n_aug), leave=False)
+        ]
         return np.stack(preds1+preds2), targs
 
     def fit_opt_sched(self, phases, cycle_save_name=None, best_save_name=None, stop_div=False, data_list=None, callbacks=None, 

fastai/lm_rnn.py

@@ -79,7 +79,9 @@ def __init__(self, ntoken, emb_sz, n_hid, n_layers, pad_token, bidir=False,
 
         self.emb_sz,self.n_hid,self.n_layers,self.dropoute = emb_sz,n_hid,n_layers,dropoute
         self.dropouti = LockedDropout(dropouti)
-        self.dropouths = nn.ModuleList([LockedDropout(dropouth) for l in range(n_layers)])
+        self.dropouths = nn.ModuleList(
+            [LockedDropout(dropouth) for _ in range(n_layers)]
+        )
 
     def forward(self, input):
         """ Invoked during the forward propagation of the RNN_Encoder module.