From 3d9281f3b6f0a50b423b8ffcb2e20e0d3a0b1b1e Mon Sep 17 00:00:00 2001 From: Sourcery AI <> Date: Wed, 26 Jul 2023 13:34:34 +0000 Subject: [PATCH] 'Refactored by Sourcery' --- 2.ASimpleNeuralNetwork/fizbuz.py | 4 ++-- 2.ASimpleNeuralNetwork/numpy_like_fizbuz.py | 6 ++--- .../ModelImplementation/bottleneck_support.py | 4 ++-- .../ModelImplementation/profile_support.py | 4 ++-- 4.ComputerVision/SemSeg/dataset.py | 4 ++-- 4.ComputerVision/SemSeg/segmentation.py | 4 ++-- .../AdvancedRNN/model.py | 3 +-- .../AdvancedRNN/train.py | 6 ++--- .../RecursiveNet/model.py | 12 ++++------ .../RecursiveNet/train.py | 2 +- 5.SequentialDataProcessing/SimpleRNN/model.py | 3 +-- 5.SequentialDataProcessing/SimpleRNN/train.py | 4 ++-- .../AutoRegressive/wavenet.py | 2 +- .../AutoRegressive/wavenet_data.py | 13 ++++------- 6.GenerativeNetworks/GAN/CycleGAN/mode.py | 19 ++++++++------- 6.GenerativeNetworks/GAN/CycleGAN/util.py | 3 +-- 6.GenerativeNetworks/GAN/SimpleGAN/model.py | 7 +++--- .../reinforcement_learning.py | 20 +++++----------- .../FizBuzFlask/controller.py | 5 +--- .../fizbuz_package/fizbuz_service.py | 23 ++++++------------- 8.PyTorchInProduction/RedisAI/run_redis.py | 5 +--- .../TorchScriptExamples/multinomial.py | 3 +-- 22 files changed, 59 insertions(+), 97 deletions(-) diff --git a/2.ASimpleNeuralNetwork/fizbuz.py b/2.ASimpleNeuralNetwork/fizbuz.py index 271f358..42b17ac 100644 --- a/2.ASimpleNeuralNetwork/fizbuz.py +++ b/2.ASimpleNeuralNetwork/fizbuz.py @@ -77,8 +77,8 @@ def forward(self, batch): for i in range(len(teX)): num = decoder(teX[i]) print( - 'Number: {} -- Actual: {} -- Prediction: {}'.format( - num, check_fizbuz(num), outli[hyp[i].max(0)[1].item()])) + f'Number: {num} -- Actual: {check_fizbuz(num)} -- Prediction: {outli[hyp[i].max(0)[1].item()]}' + ) print('Test loss: ', output.item() / len(x)) accuracy = hyp.max(1)[1] == y print('accuracy: ', accuracy.sum().item() / len(accuracy)) diff --git a/2.ASimpleNeuralNetwork/numpy_like_fizbuz.py b/2.ASimpleNeuralNetwork/numpy_like_fizbuz.py index 6e36fce..55305a8 100644 --- a/2.ASimpleNeuralNetwork/numpy_like_fizbuz.py +++ b/2.ASimpleNeuralNetwork/numpy_like_fizbuz.py @@ -32,7 +32,7 @@ b1 = torch.zeros(1, hidden_size, requires_grad=True, device=device, dtype=dtype) b2 = torch.zeros(1, output_size, requires_grad=True, device=device, dtype=dtype) -no_of_batches = int(len(trX) / batches) +no_of_batches = len(trX) // batches for epoch in range(epochs): for batch in range(no_of_batches): start = batch * batches @@ -106,8 +106,8 @@ for i in range(len(teX)): num = decoder(teX[i]) print( - 'Number: {} -- Actual: {} -- Prediction: {}'.format( - num, check_fizbuz(num), outli[hyp[i].max(0)[1].item()])) + f'Number: {num} -- Actual: {check_fizbuz(num)} -- Prediction: {outli[hyp[i].max(0)[1].item()]}' + ) print('Test loss: ', output.item() / len(x)) accuracy = hyp.max(1)[1] == y.max(1)[1] print('accuracy: ', accuracy.sum().item() / len(accuracy)) diff --git a/3.DLWorkFlow/ModelImplementation/bottleneck_support.py b/3.DLWorkFlow/ModelImplementation/bottleneck_support.py index 475edb1..08237a3 100644 --- a/3.DLWorkFlow/ModelImplementation/bottleneck_support.py +++ b/3.DLWorkFlow/ModelImplementation/bottleneck_support.py @@ -89,8 +89,8 @@ def forward(self, batch): net = FizBuzNet(input_size, 4) loss_fn = nn.MSELoss() optimizer = optim.Adam(net.parameters(), lr=lr) -x_ = x[0:10] -y_ = y[0:10] +x_ = x[:10] +y_ = y[:10] hyp = net(x_) loss = loss_fn(hyp, y_) loss.backward() diff --git a/3.DLWorkFlow/ModelImplementation/profile_support.py b/3.DLWorkFlow/ModelImplementation/profile_support.py index 539c909..3f138f0 100644 --- a/3.DLWorkFlow/ModelImplementation/profile_support.py +++ b/3.DLWorkFlow/ModelImplementation/profile_support.py @@ -91,8 +91,8 @@ def forward(self, batch): net = FizBuzNet(input_size, 4) loss_fn = nn.MSELoss() optimizer = optim.Adam(net.parameters(), lr=lr) -x_ = x[0:10] -y_ = y[0:10] +x_ = x[:10] +y_ = y[:10] with torch.autograd.profiler.profile() as prof: hyp = net(x_) diff --git a/4.ComputerVision/SemSeg/dataset.py b/4.ComputerVision/SemSeg/dataset.py index 98ff3f9..546c97f 100644 --- a/4.ComputerVision/SemSeg/dataset.py +++ b/4.ComputerVision/SemSeg/dataset.py @@ -22,12 +22,12 @@ def __init__(self, split, path): try: input_files.remove('labels') except ValueError: - raise FileNotFoundError("Couldn't find 'labels' folder in {}".format(path)) + raise FileNotFoundError(f"Couldn't find 'labels' folder in {path}") self.files = [] for file in input_files: name, ext = os.path.splitext(file) input_file = os.path.join(inputdir_path, file) - label_file = os.path.join(labledir_path, '{}_L{}'.format(name, ext)) + label_file = os.path.join(labledir_path, f'{name}_L{ext}') self.files.append({'input': input_file, 'label': label_file}) mean = [104.00699, 116.66877, 122.67892] # found from meetshah1995/pytorch-semseg std = [255, 255, 255] diff --git a/4.ComputerVision/SemSeg/segmentation.py b/4.ComputerVision/SemSeg/segmentation.py index 92c01ee..6190831 100644 --- a/4.ComputerVision/SemSeg/segmentation.py +++ b/4.ComputerVision/SemSeg/segmentation.py @@ -35,11 +35,11 @@ def create_image(out): """ Creating image from the outbatch """ img = out[0].max(0)[1].data.cpu().numpy() - misc.imsave('{}.png'.format(time.time()), img) + misc.imsave(f'{time.time()}.png', img) def save_model(model): - torch.save(model.state_dict(), '{}.pth'.format(time.time())) + torch.save(model.state_dict(), f'{time.time()}.pth') for epoch in range(epochs): diff --git a/5.SequentialDataProcessing/AdvancedRNN/model.py b/5.SequentialDataProcessing/AdvancedRNN/model.py index 6a49e1f..9297ed3 100644 --- a/5.SequentialDataProcessing/AdvancedRNN/model.py +++ b/5.SequentialDataProcessing/AdvancedRNN/model.py @@ -65,5 +65,4 @@ def forward(self, batch): hypo_embed = self.embed(batch.hypothesis) premise = self.encoder(prem_embed) hypothesis = self.encoder(hypo_embed) - scores = self.classifier((premise, hypothesis)) - return scores + return self.classifier((premise, hypothesis)) diff --git a/5.SequentialDataProcessing/AdvancedRNN/train.py b/5.SequentialDataProcessing/AdvancedRNN/train.py index 89a1a47..10d3f98 100644 --- a/5.SequentialDataProcessing/AdvancedRNN/train.py +++ b/5.SequentialDataProcessing/AdvancedRNN/train.py @@ -63,10 +63,10 @@ best_dev_acc = -1 train_iter.repeat = False -for epoch in range(epochs): +for _ in range(epochs): train_iter.init_epoch() n_correct, n_total = 0, 0 - for batch_idx, batch in enumerate(train_iter): + for batch in train_iter: model.train() opt.zero_grad() iterations += 1 @@ -83,7 +83,7 @@ model.eval() dev_iter.init_epoch() n_dev_correct, dev_loss = 0, 0 - for dev_batch_idx, dev_batch in enumerate(dev_iter): + for dev_batch in dev_iter: answer = model(dev_batch) n_dev_correct += (torch.max(answer, 1) [1].view(dev_batch.label.size()) == dev_batch.label).sum() diff --git a/5.SequentialDataProcessing/RecursiveNet/model.py b/5.SequentialDataProcessing/RecursiveNet/model.py index a3c54ee..8abd8b9 100644 --- a/5.SequentialDataProcessing/RecursiveNet/model.py +++ b/5.SequentialDataProcessing/RecursiveNet/model.py @@ -15,9 +15,7 @@ def bundle(lstm_iter): if lstm_iter is None: return None lstm_iter = tuple(lstm_iter) - if lstm_iter[0] is None: - return None - return torch.cat(lstm_iter, 0).chunk(2, 1) + return None if lstm_iter[0] is None else torch.cat(lstm_iter, 0).chunk(2, 1) class Bottle(nn.Module): @@ -60,8 +58,7 @@ def forward(self, left_in, right_in, tracking=None): lstm_in += self.right(right[0]) if hasattr(self, 'track'): lstm_in += self.track(tracking[0]) - out = unbundle(tree_lstm(left[1], right[1], lstm_in)) - return out + return unbundle(tree_lstm(left[1], right[1], lstm_in)) class Tracker(nn.Module): @@ -172,7 +169,7 @@ def __init__(self, config): mlp_in_size = 4 * feat_in_size mlp = [nn.Linear(mlp_in_size, config.d_mlp), self.relu, nn.BatchNorm1d(config.d_mlp), self.mlp_dropout] - for i in range(config.n_mlp_layers - 1): + for _ in range(config.n_mlp_layers - 1): mlp.extend([nn.Linear(config.d_mlp, config.d_mlp), self.relu, nn.BatchNorm1d(config.d_mlp), self.mlp_dropout]) mlp.append(nn.Linear(config.d_mlp, config.d_out)) @@ -190,5 +187,4 @@ def forward(self, batch): prem_trans = hypo_trans = None premise = self.encoder(prem_embed, prem_trans) hypothesis = self.encoder(hypo_embed, hypo_trans) - scores = self.out(self.merger(premise, hypothesis)) - return scores + return self.out(self.merger(premise, hypothesis)) diff --git a/5.SequentialDataProcessing/RecursiveNet/train.py b/5.SequentialDataProcessing/RecursiveNet/train.py index 7a727e4..1a4e41e 100644 --- a/5.SequentialDataProcessing/RecursiveNet/train.py +++ b/5.SequentialDataProcessing/RecursiveNet/train.py @@ -112,7 +112,7 @@ class Config: model.eval() dev_iter.init_epoch() n_dev_correct, dev_loss = 0, 0 - for dev_batch_idx, dev_batch in enumerate(dev_iter): + for dev_batch in dev_iter: answer = model(dev_batch) n_dev_correct += (torch.max( answer, 1)[1].view(dev_batch.label.size()).data == dev_batch.label.data).sum() diff --git a/5.SequentialDataProcessing/SimpleRNN/model.py b/5.SequentialDataProcessing/SimpleRNN/model.py index 9c8b26e..eb61f1e 100644 --- a/5.SequentialDataProcessing/SimpleRNN/model.py +++ b/5.SequentialDataProcessing/SimpleRNN/model.py @@ -79,5 +79,4 @@ def forward(self, batch): hypo_embed = self.embed(batch.hypothesis) premise = self.encoder(prem_embed) hypothesis = self.encoder(hypo_embed) - scores = self.classifier((premise, hypothesis)) - return scores + return self.classifier((premise, hypothesis)) diff --git a/5.SequentialDataProcessing/SimpleRNN/train.py b/5.SequentialDataProcessing/SimpleRNN/train.py index 8a0e2c0..84cae94 100644 --- a/5.SequentialDataProcessing/SimpleRNN/train.py +++ b/5.SequentialDataProcessing/SimpleRNN/train.py @@ -73,7 +73,7 @@ def init_weights(m): train_iter.repeat = False model.train() -for epoch in range(epochs): +for _ in range(epochs): train_iter.init_epoch() n_correct, n_total = 0, 0 for batch_idx, batch in enumerate(train_iter): @@ -98,7 +98,7 @@ def init_weights(m): model.eval() dev_iter.init_epoch() n_dev_correct, dev_loss = 0, 0 - for dev_batch_idx, dev_batch in enumerate(dev_iter): + for dev_batch in dev_iter: answer = model(dev_batch) n_dev_correct += (torch.max( answer, 1)[1].view( diff --git a/6.GenerativeNetworks/AutoRegressive/wavenet.py b/6.GenerativeNetworks/AutoRegressive/wavenet.py index d2cf51c..7b38d42 100644 --- a/6.GenerativeNetworks/AutoRegressive/wavenet.py +++ b/6.GenerativeNetworks/AutoRegressive/wavenet.py @@ -29,7 +29,7 @@ class ResidualStack(torch.nn.Module): def __init__(self, layer_size, stack_size, res_channels, skip_channels): super().__init__() self.res_blocks = torch.nn.ModuleList() - for s in range(stack_size): + for _ in range(stack_size): for l in range(layer_size): dilation = 2 ** l block = ResidualBlock(res_channels, skip_channels, dilation) diff --git a/6.GenerativeNetworks/AutoRegressive/wavenet_data.py b/6.GenerativeNetworks/AutoRegressive/wavenet_data.py index efdd53c..a737581 100644 --- a/6.GenerativeNetworks/AutoRegressive/wavenet_data.py +++ b/6.GenerativeNetworks/AutoRegressive/wavenet_data.py @@ -28,9 +28,7 @@ def one_hot_encode(data, channels=256): def one_hot_decode(data, axis=1): - decoded = np.argmax(data, axis=axis) - - return decoded + return np.argmax(data, axis=axis) def mu_law_encode(audio, quantization_channels=256): @@ -55,9 +53,7 @@ def mu_law_decode(output, quantization_channels=256): mu = float(quantization_channels - 1) expanded = (output / quantization_channels) * 2. - 1 - waveform = np.sign(expanded) * (np.exp(np.abs(expanded) * np.log(mu + 1)) - 1) / mu - - return waveform + return np.sign(expanded) * (np.exp(np.abs(expanded) * np.log(mu + 1)) - 1) / mu class Dataset(data.Dataset): @@ -69,7 +65,7 @@ def __init__(self, data_dir, sample_rate=16000, in_channels=256, trim=True): self.trim = trim self.root_path = data_dir - self.filenames = [x for x in sorted(os.listdir(data_dir))] + self.filenames = list(sorted(os.listdir(data_dir))) def __getitem__(self, index): filepath = os.path.join(self.root_path, self.filenames[index]) @@ -116,8 +112,7 @@ def __init__(self, data_dir, receptive_fields, self.collate_fn = self._collate_fn def calc_sample_size(self, audio): - return self.sample_size if len(audio[0]) >= self.sample_size\ - else len(audio[0]) + return min(len(audio[0]), self.sample_size) @staticmethod def _variable(data): diff --git a/6.GenerativeNetworks/GAN/CycleGAN/mode.py b/6.GenerativeNetworks/GAN/CycleGAN/mode.py index 9bde94f..6e55961 100644 --- a/6.GenerativeNetworks/GAN/CycleGAN/mode.py +++ b/6.GenerativeNetworks/GAN/CycleGAN/mode.py @@ -26,8 +26,8 @@ def __init__(self, root, transforms_=None, unaligned=False, mode='train'): self.transform = transforms.Compose(transforms_) self.unaligned = unaligned - self.files_A = sorted(glob.glob(os.path.join(root, '%sA' % mode) + '/*.*')) - self.files_B = sorted(glob.glob(os.path.join(root, '%sB' % mode) + '/*.*')) + self.files_A = sorted(glob.glob(f"{os.path.join(root, f'{mode}A')}/*.*")) + self.files_B = sorted(glob.glob(f"{os.path.join(root, f'{mode}B')}/*.*")) def __getitem__(self, index): item_A = self.transform(Image.open(self.files_A[index % len(self.files_A)])) @@ -223,13 +223,12 @@ def push_and_pop(self, data): if len(self.data) < self.max_size: self.data.append(element) to_return.append(element) + elif random.uniform(0, 1) > 0.5: + i = random.randint(0, self.max_size - 1) + to_return.append(self.data[i].clone()) + self.data[i] = element else: - if random.uniform(0, 1) > 0.5: - i = random.randint(0, self.max_size - 1) - to_return.append(self.data[i].clone()) - self.data[i] = element - else: - to_return.append(element) + to_return.append(element) return torch.cat(to_return) @@ -316,8 +315,8 @@ def weights_init_normal(m): ###### Training ###### -for epoch in range(opt.epoch, opt.n_epochs): - for i, batch in enumerate(dataloader): +for _ in range(opt.epoch, opt.n_epochs): + for batch in dataloader: # Set model input real_A = input_A.copy_(batch['A']) real_B = input_B.copy_(batch['B']) diff --git a/6.GenerativeNetworks/GAN/CycleGAN/util.py b/6.GenerativeNetworks/GAN/CycleGAN/util.py index f47369d..2551a5a 100644 --- a/6.GenerativeNetworks/GAN/CycleGAN/util.py +++ b/6.GenerativeNetworks/GAN/CycleGAN/util.py @@ -26,5 +26,4 @@ def get_args(): "--cuda", type=bool, default=torch.cuda.is_available(), help='CUDA availability check') parser.add_argument('--size', type=int, default=256, help='crop to this size') - args = parser.parse_args(args=[]) - return args + return parser.parse_args(args=[]) diff --git a/6.GenerativeNetworks/GAN/SimpleGAN/model.py b/6.GenerativeNetworks/GAN/SimpleGAN/model.py index 8c65cd2..fe012da 100644 --- a/6.GenerativeNetworks/GAN/SimpleGAN/model.py +++ b/6.GenerativeNetworks/GAN/SimpleGAN/model.py @@ -11,7 +11,7 @@ def mnist_data(): compose = transforms.Compose( [transforms.ToTensor(), transforms.Normalize((0.5,), (0.5,))]) - out_dir = '{}/dataset'.format(DATA_FOLDER) + out_dir = f'{DATA_FOLDER}/dataset' return datasets.MNIST(root=out_dir, train=True, transform=compose, download=True) @@ -182,9 +182,8 @@ def train_generator(optimizer, fake_data): test_noise = noise(num_test_samples) -for epoch in range(num_epochs): - for n_batch, (real_batch, _) in enumerate(data_loader): - +for _ in range(num_epochs): + for real_batch, _ in data_loader: # 1. Train Discriminator real_data = images_to_vectors(real_batch).to(device) # Generate fake data diff --git a/7.ReinforcementLearning/reinforcement_learning.py b/7.ReinforcementLearning/reinforcement_learning.py index 535101b..40c85b6 100644 --- a/7.ReinforcementLearning/reinforcement_learning.py +++ b/7.ReinforcementLearning/reinforcement_learning.py @@ -87,16 +87,12 @@ def select_action(state): steps_done += 1 sample = random.random() - if sample > eps_threshold: - - # freeze the network and get predictions - with torch.no_grad(): - return policy_net(state).max(1)[1].view(1, 1) - - else: - + if sample <= eps_threshold: # select random action return torch.tensor([[random.randrange(2)]], device=device, dtype=torch.long) + # freeze the network and get predictions + with torch.no_grad(): + return policy_net(state).max(1)[1].view(1, 1) Transition = namedtuple('Transition', @@ -179,7 +175,7 @@ def optimize_model(): current_screen = get_screen() state = current_screen - last_screen - for t in count(): # for each timestep in an episode + for _ in count(): # Select action for the given state and get rewards action = select_action(state) _, reward, done, _ = env.step(action.item()) @@ -188,11 +184,7 @@ def optimize_model(): # Observe new state last_screen = current_screen current_screen = get_screen() - if not done: - next_state = current_screen - last_screen - else: - next_state = None - + next_state = current_screen - last_screen if not done else None # Store the transition in memory memory.push(state, action, next_state, reward) diff --git a/8.PyTorchInProduction/FizBuzFlask/controller.py b/8.PyTorchInProduction/FizBuzFlask/controller.py index 8ff8822..e7ef25b 100644 --- a/8.PyTorchInProduction/FizBuzFlask/controller.py +++ b/8.PyTorchInProduction/FizBuzFlask/controller.py @@ -11,10 +11,7 @@ def get_readable_output(input_num, prediction): 0: 'FizBuz', 1: 'Buz', 2: 'Fiz'} - if prediction == 3: - return input_num - else: - return input_output_map[prediction] + return input_num if prediction == 3 else input_output_map[prediction] def binary_encoder(): diff --git a/8.PyTorchInProduction/FizBuzONNX/fizbuz_package/fizbuz_service.py b/8.PyTorchInProduction/FizBuzONNX/fizbuz_package/fizbuz_service.py index 29cda1f..e952ab0 100644 --- a/8.PyTorchInProduction/FizBuzONNX/fizbuz_package/fizbuz_service.py +++ b/8.PyTorchInProduction/FizBuzONNX/fizbuz_package/fizbuz_service.py @@ -28,10 +28,7 @@ def get_readable_output(self, input_num, prediction): 0: 'FizBuz', 1: 'Buz', 2: 'Fiz'} - if prediction == 3: - return input_num - else: - return input_output_map[prediction] + return input_num if prediction == 3 else input_output_map[prediction] def initialize(self, context): # todo - check batch size and read it from the message @@ -50,8 +47,6 @@ def initialize(self, context): model_files_prefix = self.get_model_files_prefix(context) - data_names = [] - data_shapes = [] input_data = self.signature["inputs"][0] data_name = input_data["data_name"] data_shape = input_data["data_shape"] @@ -61,10 +56,9 @@ def initialize(self, context): for idx in range(len(data_shape)): if data_shape[idx] == 0: data_shape[idx] = 1 - data_names.append(data_name) - data_shapes.append((data_name, tuple(data_shape))) - - checkpoint_prefix = "{}/{}".format(model_dir, model_files_prefix) + data_names = [data_name] + data_shapes = [(data_name, tuple(data_shape))] + checkpoint_prefix = f"{model_dir}/{model_files_prefix}" # Load MXNet module self.mxnet_ctx = mx.cpu() if gpu_id is None else mx.gpu(gpu_id) sym, arg_params, aux_params = mx.model.load_checkpoint( @@ -82,11 +76,9 @@ def preprocess(self, batch): # todo - assert batch size param_name = self.signature['inputs'][0]['data_name'] - data = batch[0].get('body').get(param_name) - if data: + if data := batch[0].get('body').get(param_name): self.input = data + 1 - tensor = mx.nd.array([self.binary_encoder(self.input, input_size=10)]) - return tensor + return mx.nd.array([self.binary_encoder(self.input, input_size=10)]) self.error = 'InvalidData' def inference(self, model_input): @@ -114,8 +106,7 @@ def postprocess(self, inference_output): prediction = self.get_readable_output( self.input, int(inference_output[0].argmax(1).asscalar())) - out = [{'next_number': prediction}] - return out + return [{'next_number': prediction}] def handle(self, data, context): try: diff --git a/8.PyTorchInProduction/RedisAI/run_redis.py b/8.PyTorchInProduction/RedisAI/run_redis.py index e04c0e9..164193b 100644 --- a/8.PyTorchInProduction/RedisAI/run_redis.py +++ b/8.PyTorchInProduction/RedisAI/run_redis.py @@ -13,10 +13,7 @@ def get_readable_output(input_num, prediction): 0: 'FizBuz', 1: 'Buz', 2: 'Fiz'} - if prediction == 3: - return input_num - else: - return input_output_map[prediction] + return input_num if prediction == 3 else input_output_map[prediction] r = redis.Redis() diff --git a/8.PyTorchInProduction/TorchScriptExamples/multinomial.py b/8.PyTorchInProduction/TorchScriptExamples/multinomial.py index 26a1351..0e830a1 100644 --- a/8.PyTorchInProduction/TorchScriptExamples/multinomial.py +++ b/8.PyTorchInProduction/TorchScriptExamples/multinomial.py @@ -1,4 +1,3 @@ def post_processing(output): output_dist = output.squeeze().div(0.8).exp() - prob = torch.multinomial(output_dist, 2) - return prob + return torch.multinomial(output_dist, 2)