Cicada init

.cicada/my-pipeline.ts

@@ -0,0 +1,20 @@
+import { Job, Pipeline } from "https://deno.land/x/cicada@v0.1.32/lib.ts";
+
+const job = new Job({
+  name: "My First Job",
+  image: "ubuntu:22.04",
+  steps: [
+    {
+      name: "Print a message",
+      run: "echo Hello, world!",
+    },
+    {
+      name: "Run a js function",
+      run: () => {
+        console.log("Hello from js");
+      },
+    },
+  ],
+});
+
+export default new Pipeline([job]);

.vscode/settings.json

@@ -0,0 +1,3 @@
+{
+  "deno.enablePaths": [".cicada"]
+}

apps/scraper/index.py

@@ -34,9 +34,9 @@ def upload_to_postgres(file):
 
 if __name__ == "__main__":
     # If output file is not found then refresh data
-    if glob('./output/*.csv'):
-        print("Output found will not refresh data...")
-    else:
-        print("Output not found refreshing data...")
-        data_file = scrape_website()
-        upload_to_postgres(data_file)
+    # if glob('./output/*.csv'):
+    #     print("Output found will not refresh data...")
+    # else:
+    #     print("Output not found refreshing data...")
+    #     data_file = scrape_website()
+    upload_to_postgres("./output/restaurants1658265034.csv")

apps/scraper/scraper.py

@@ -29,7 +29,7 @@ def __init__(self):
         self.restaurants = []
         self.url = base_url + query + '1'
         self.content = bs(requests.get(self.url).content, 'html.parser')
-        self.delay_rate = 0.15
+        self.delay_rate = 0.02
         self.total_pages = self.get_total_pages(self.content)
         self.cur_year = date.today().year