add example 15: process csv metadata into columns

Author: NicolasCARPi

examples/15-process-csv-metadata-into-csv-column.py

@@ -0,0 +1,38 @@
+#!/usr/bin/env python
+
+# In this script, we use a CSV that has been created from eLabFTW
+# Then loop over all the entries, extract the metadata, and create another CSV
+# where each metadata extra field is a column
+import json
+import csv
+
+# First we import a CSV file that has been created from eLabFTW export function
+# Important: use encoding parameter to decode the CSV properly (https://stackoverflow.com/a/49150749)
+with open('data/antibodies-export-from-elab.csv', encoding='utf-8-sig') as csvfile:
+    # Use the DictReader function to get Dict objects back
+    csvreader = csv.DictReader(csvfile)
+    # We want all rows as a list of rows
+    rows = []
+    for row in csvreader:
+        rows.append(row)
+
+# This Set will be needed when exporting to csv
+fieldnames = set()
+for entry in rows:
+    # Let's display for each row, the title and its corresponding metadata
+    print(f"{entry.get('title')} => {entry.get('metadata')}")
+    # Now let's process that metadata
+    metadata = json.loads(entry.get('metadata'))
+    # Extract fields from 'extra_fields' and add them to the item dictionary
+    for key, value in metadata.get('extra_fields').items():
+        entry[key] = value.get('value')
+    # Remove the original JSON metadata
+    del entry['metadata']
+    fieldnames.update(entry.keys())
+
+output = 'output.csv'
+print(f"[INFO] Writing output to {output}")
+with open(output, 'w') as f:
+    writer = csv.DictWriter(f, fieldnames=fieldnames)
+    writer.writeheader()
+    writer.writerows(rows)

examples/README.md

@@ -61,3 +61,7 @@ Get a list of events (booked slots) for the current team.
 # [14-fix-user-exp-permissions.py](./14-fix-user-exp-permissions.py)
 
 Look for all experiments of a given user and adjust permissions of their experiments.
+
+# [15-process-csv-metadata-into-csv-column.py](./15-process-csv-metadata-into-csv-column.py)
+
+Take a CSV file created from eLabFTW export function, and for each row, grab the metadata, and create a new CSV where each Extra Field in the metadata is now a column.

examples/data/antibodies-export-from-elab.csv

@@ -0,0 +1,4 @@
+id,date,title,body,category,category_title,category_color,status,status_title,status_color,custom_id,elabid,rating,url,metadata,tags
+172,2024-03-28,Anti-Tubulin,,8,Antibody,C24F3D,3,In stock,c1e818,,20240328-cf7e261212d1d42bd58b36c213d0fb822043c2fd,0,https://elab.local:3148/database.php?mode=view&id=172,"{""extra_fields"": {""Host"": {""type"": ""select"", ""value"": ""Rabbit"", ""options"": [""Rabbit"", ""Mouse"", ""Goat"", ""Chicken""]}, ""Storage temperature"": {""type"": ""select"", ""value"": ""+4 °C"", ""options"": [""+4 °C"", ""-20 °C"", ""-80 °C""]}, ""Dilution to use for IF"": {""type"": ""text"", ""value"": ""1:100""}, ""Dilution to use for WB"": {""type"": ""text"", ""value"": ""1:1000""}, ""Number of aliquots left"": {""type"": ""number"", ""value"": ""12"", ""description"": ""Make sure to decrement this when using one!""}}}",
+174,2024-03-28,Anti-V5,,8,Antibody,C24F3D,3,In stock,c1e818,,20240328-0f967dd28c85db06f1b609d5ae468f1a04e68ca8,0,https://elab.local:3148/database.php?mode=view&id=174,"{""extra_fields"": {""Host"": {""type"": ""select"", ""value"": ""Rabbit"", ""options"": [""Rabbit"", ""Mouse"", ""Goat"", ""Chicken""]}, ""Storage temperature"": {""type"": ""select"", ""value"": ""+4 °C"", ""options"": [""+4 °C"", ""-20 °C"", ""-80 °C""]}, ""Dilution to use for IF"": {""type"": ""text"", ""value"": ""1:50""}, ""Dilution to use for WB"": {""type"": ""text"", ""value"": ""1:10000""}, ""Number of aliquots left"": {""type"": ""number"", ""value"": ""3"", ""description"": ""Make sure to decrement this when using one!""}}}",
+163,2024-03-28,Anti-Actin,,8,Antibody,C24F3D,3,In stock,c1e818,,20240328-7fa3f598c020da1593be7e07e3fdf78f8f4e5150,0,https://elab.local:3148/database.php?mode=view&id=163,"{""extra_fields"": {""Host"": {""type"": ""select"", ""value"": ""Rabbit"", ""options"": [""Rabbit"", ""Mouse"", ""Goat"", ""Chicken""]}, ""Storage temperature"": {""type"": ""select"", ""value"": ""+4 °C"", ""options"": [""+4 °C"", ""-20 °C"", ""-80 °C""]}, ""Dilution to use for IF"": {""type"": ""text"", ""value"": ""1:75""}, ""Dilution to use for WB"": {""type"": ""text"", ""value"": ""1:500""}, ""Number of aliquots left"": {""type"": ""number"", ""value"": ""10"", ""description"": ""Make sure to decrement this when using one!""}}}",