Merge pull request #11 from anusii/dc/10_use_inherit_key

dc/10 use inherit key

Author: gjwgit

read_encrypted.py

@@ -1,17 +1,22 @@
 import os
+import sys
 import argparse
 
 from getpass import getpass
 from base64 import b64decode
+from urllib.parse import urlparse
 
-from pod_helper import (
+from solidpod_helper import (
     gen_master_key,
+    gen_verify_key,
     decrypt,
     parse_ttl,
     path_pred,
     iv_pred,
-    session_key_pred,
-    data_pred,
+    verify_key_pred,
+    indi_key_pred,
+    inherit_key_pred,
+    enc_data_pred,
     server_path,
 )
 
@@ -30,39 +35,75 @@
     app_name = items[1]
     assert items[2] == 'data'
     app_path = f'{server_path}{pod_name}/{app_name}'
+    relative_file_path = '/'.join(items[1:])
 
     security_key_str = getpass(prompt='Security Key: ')
     master_key = gen_master_key(security_key_str)
+    verify_key = gen_verify_key(security_key_str)
 
-    # Parsing ind-keys.ttl file
+    # Verify security key
 
-    key_path = f'{app_path}/encryption/ind-keys.ttl'
-    result = parse_ttl(key_path)
-    key_map = {v[path_pred][0]: {iv_pred: v[iv_pred][0], session_key_pred: v[session_key_pred][0]} for k, v in result.items() if iv_pred in v}
-    file_key = '/'.join(items[1:])
-    session_key_ct_b64 = key_map[file_key][session_key_pred]
-    session_key_iv_b64 = key_map[file_key][iv_pred]
+    enc_key_map = parse_ttl(f'{app_path}/encryption/enc-keys.ttl')
+    verify_key_stored = list(enc_key_map.items())[0][1][verify_key_pred]
+    if verify_key.decode('utf-8') != verify_key_stored:
+        print('ERROR: Incorrect security key (verification failed).')
+        sys.exit(0)
 
     # Parse data .ttl file
 
-    result = parse_ttl(file_path)
-    data_key = list(result.keys())[0]
-    data_map = {iv_pred: result[data_key][iv_pred][0], data_pred: result[data_key][data_pred][0]}
-    data_ct_b64 = data_map[data_pred]
-    data_iv_b64 = data_map[iv_pred]
+    with open(file_path) as fd:
+        file_content = fd.read()
 
-    # Decrypt session key
+    _map = parse_ttl(ttl_str=file_content)
 
-    session_key_ct = b64decode(session_key_ct_b64)
-    session_key_iv = b64decode(session_key_iv_b64)
-    session_key_b64 = decrypt(session_key_ct, master_key, session_key_iv)
-    session_key = b64decode(session_key_b64)
+    def exit():
+        print(f'WARN: File "{args.filepath}" is not encrypted by solidpod, return raw content\n{"-"*20}')
+        print(file_content)
+        sys.exit(0)
+    
+    if len(_map) != 1:
+        exit()
+
+    file_url, data_map = list(_map.items())[0]
+
+    if (path_pred not in data_map) or (data_map[path_pred] != relative_file_path):
+        exit()
+
+    if (iv_pred not in data_map) or (enc_data_pred not in data_map):
+        exit()
+
+    data_ct = b64decode(data_map[enc_data_pred])
+    data_iv = b64decode(data_map[iv_pred])
+
+    # Retrieve encryption key and IV
+
+    key_map = parse_ttl(f'{app_path}/encryption/ind-keys.ttl')
+    indi_key_ct = None
+    indi_key_iv = None
+    if file_url in key_map:
+        indi_key_ct = b64decode(key_map[file_url][indi_key_pred])
+        indi_key_iv = b64decode(key_map[file_url][iv_pred])
+    elif inherit_key_pred in data_map:
+        r = urlparse(file_url)
+        server_url = f'{r.scheme}://{r.netloc}'
+        inherit_key_url = '/'.join([server_url, pod_name, data_map[inherit_key_pred]])
+        if inherit_key_url in key_map:
+            indi_key_ct = b64decode(key_map[inherit_key_url][indi_key_pred])
+            indi_key_iv = b64decode(key_map[inherit_key_url][iv_pred])
+    else:
+        pass
+
+    if indi_key_ct is None or indi_key_iv is None:
+        exit()
+
+    # Decrypt individual key
+
+    indi_key = b64decode(decrypt(indi_key_ct, master_key, indi_key_iv))
 
     # Decrypt data
 
-    data_ct = b64decode(data_ct_b64)
-    data_iv = b64decode(data_iv_b64)
-    plain_text = decrypt(data_ct, session_key, data_iv)
+    plain_text = decrypt(data_ct, indi_key, data_iv)
 
+    print('-'*20)
     print(plain_text)
 

pod_helper.py solidpod_helper.pypod_helper.py renamed to solidpod_helper.py

@@ -1,3 +1,4 @@
+import sys
 import hashlib
 from base64 import (
     b64decode,
@@ -15,40 +16,55 @@
     unpad,
 )
 
-
 separator = '#'
 path_pred = 'path'
 iv_pred = 'iv'
-session_key_pred = 'sessionKey'
-data_pred = 'encData'
+verify_key_pred = 'encKey'
+private_key_pred = 'prvKey'
+indi_key_pred = 'sessionKey'
+enc_data_pred = 'encData'
+inherit_key_pred = 'inheritKeyFrom'
 server_path = '/opt/solid/server/'
 apps_terms = 'https://solidcommunity.au/predicates/terms#';
 
-def parse_ttl(fname):
-    # Parse a .ttl file into a dictionary
+def parse_ttl(fname=None, ttl_str=None):
+    # Parse a .ttl file or its content (RDF data string) into a dictionary
+    if fname is None and ttl_str is None:
+        print(f'ERROR: either file name or RDF data should be provided')
+        sys.exit(1)
+
     g = Graph()
-    g.parse(fname)
+    if fname is not None:
+        g.parse(fname)
+    elif ttl_str is not None:
+        g.parse(data=ttl_str)
+    else:
+        pass
 
     tripleMap = dict()
     for sub, pre, obj in g:
-        s = sub.lower() if isinstance(sub, URIRef) else sub
-        p = pre.split('#')[-1]
-        o = obj.split('#')[-1]
+        s = str(sub) if isinstance(sub, URIRef) else sub
+        p = pre.split(separator)[-1]
+        o = obj.split(separator)[-1]
         if s in tripleMap:
             d = tripleMap[s]
             if p in d:
-                d[p].add(o)
+                d[p] = d.add(o)
             else:
                 d[p] = [o]
         else:
             tripleMap[s] = {p: [o]}
-    return tripleMap
+    # convert triple (s, p, o) to (s, p, o[0]) if o is a list with only one element
+    return {s: {p: o[0] if len(o) == 1 else o for p, o in tripleMap[s].items()} for s in tripleMap.keys()}
 
 
 def gen_master_key(security_key_str):
     # Generate master key from security key string as per `solidpod`
     return hashlib.sha256(security_key_str.encode('utf-8')).hexdigest()[:32].encode('utf-8')
 
+def gen_verify_key(security_key_str):
+    # Generate verification key from security key string as per `solidpod`
+    return hashlib.sha224(security_key_str.encode('utf-8')).hexdigest()[:32].encode('utf-8')
 
 def encrypt(data_str, key, iv):
     # Encrypt the input string `data_str` using AES with
@@ -66,6 +82,11 @@ def encrypt(data_str, key, iv):
 
 
 def decrypt(data_ct, key, iv):
+    # Decrypt the input string `data_ct` using AES with
+    # - key (bytes): encryption key
+    # - iv (bytes): nonce and initial value
+    # Returns the plaintext string
+
     # CTR mode is also known as segmented integer counter (SIC) mode, as per
     # https://en.wikipedia.org/wiki/Block_cipher_mode_of_operation#Counter_(CTR)
     # It seems the first half of `iv' should be used as `nonce', and the latter half for `initial_value'

write_encrypted.py

@@ -2,6 +2,7 @@
 import argparse
 
 from getpass import getpass
+from urllib.parse import urlparse
 from Cryptodome.Random import get_random_bytes
 from base64 import (
     b64decode,
@@ -11,65 +12,79 @@
     Graph,
     URIRef,
 )
-from pod_helper import (
+from solidpod_helper import (
     gen_master_key,
+    gen_verify_key,
     encrypt,
+    parse_ttl,
     apps_terms,
     path_pred,
     iv_pred,
-    session_key_pred,
-    data_pred,
+    verify_key_pred,
+    indi_key_pred,
+    enc_data_pred,
     server_path,
 )
 
 
-def upload_file(file_content, file_path, server_url, master_key):
+def upload_file(file_url, file_content, master_key):
     # Encrypt file content
 
     print('Encrypt content ...')
-    session_key = get_random_bytes(32)
+    indi_key = get_random_bytes(32)
     data_iv = get_random_bytes(16)
-    enc_data_b64 = encrypt(file_content, session_key, data_iv)
+    enc_data_b64 = encrypt(file_content, indi_key, data_iv)
 
     # Write encrypted content to file in POD
 
     print('Write encrypted content ...')
+
+    r = urlparse(file_url)
+    items = r.path.split('/')[1:]  # r.path: '/pod_name/app_name/data/data_file_path'
+    assert len(items) >= 4
+    assert items[2] == 'data'
+    relative_file_path = '/'.join(items[1:])
     g = Graph()
-    file_url = f'{server_url}{file_path}'
     data_iv_b64 = b64encode(data_iv).decode('ascii')
-    query = f'INSERT DATA {{<{file_url}> <{apps_terms}{path_pred}> "{file_path}"; ' + \
+    query = f'INSERT DATA {{<{file_url}> <{apps_terms}{path_pred}> "{relative_file_path}"; ' + \
             f'<{apps_terms}{iv_pred}> "{data_iv_b64}"; ' + \
-            f'<{apps_terms}{data_pred}> "{enc_data_b64}".}};'
+            f'<{apps_terms}{enc_data_pred}> "{enc_data_b64}".}};'
     g.update(query)
     ttl_str = g.serialize(format='turtle')
-    abs_path = f'{server_path}{file_path}'
+    abs_path = f'{server_path}{"/".join(items)}'
     with open(abs_path, 'w') as f:
         f.write(ttl_str)
 
     # Add session key to ind-keys.ttl
 
     print('Add encrypted session key to ind-keys.ttl ...')
-    add_session_key(file_path, server_url, session_key, master_key)
+    add_indi_key(file_url, indi_key, master_key)
+
+def add_indi_key(file_url, indi_key, master_key):
+    # Add encrypted individual key to ind-keys.ttl
 
-def add_session_key(file_path, server_url, session_key, master_key):
-    # Add encrypted session key to ind-keys.ttl
+    # Encrypt the individual key
 
-    # Encrypt the session key
-    session_key_iv = get_random_bytes(16)
-    session_key_b64 = b64encode(session_key).decode('ascii')
-    enc_session_key_b64 = encrypt(session_key_b64, master_key, session_key_iv)
+    indi_key_iv = get_random_bytes(16)
+    indi_key_b64 = b64encode(indi_key).decode('ascii')
+    enc_indi_key_b64 = encrypt(indi_key_b64, master_key, indi_key_iv)
 
     # Parse the ind-keys.ttl file
-    items = file_path.split('/')
+
+    r = urlparse(file_url)
+    items = r.path.split('/')[1:]  # r.path: '/pod_name/app_name/data/data_file_path'
     assert len(items) >= 4
     pod_name = items[0]
     app_name = items[1]
+    assert items[2] == 'data'
+    server_url = f'{r.scheme}://{r.netloc}'
+    relative_file_path = '/'.join(items[1:])
     ind_key_path = f'{server_path}{pod_name}/{app_name}/encryption/ind-keys.ttl'
     g = Graph()
     g.parse(ind_key_path)
 
     # Replace file:///POD_DIR prefix with server URL
-    file_url = f'{server_url}{file_path}'
+
     for s, p, o in g:
         prefix = f'file://{server_path}'
         if str(s).startswith(prefix):
@@ -80,15 +95,16 @@ def add_session_key(file_path, server_url, session_key, master_key):
         if str(s) == file_url:
             g.remove((s, p, o))
 
-    # Add encrypted session key to ind-keys.ttl 
-    session_key_iv_b64 = b64encode(session_key_iv).decode('ascii')
-    fpath = '/'.join(items[1:])
-    query = f'INSERT DATA {{<{file_url}> <{apps_terms}{path_pred}> "{fpath}"; ' + \
-            f'<{apps_terms}{iv_pred}> "{session_key_iv_b64}"; ' + \
-            f'<{apps_terms}{session_key_pred}> "{enc_session_key_b64}".}};'
+    # Add encrypted individual key to ind-keys.ttl
+
+    indi_key_iv_b64 = b64encode(indi_key_iv).decode('ascii')
+    query = f'INSERT DATA {{<{file_url}> <{apps_terms}{path_pred}> "{relative_file_path}"; ' + \
+            f'<{apps_terms}{iv_pred}> "{indi_key_iv_b64}"; ' + \
+            f'<{apps_terms}{indi_key_pred}> "{enc_indi_key_b64}".}};'
     g.update(query)
 
     # Write back ind-keys.ttl
+
     ttl_str = g.serialize(format='turtle', base=server_url)
     with open(ind_key_path, 'w') as f:
         f.write(ttl_str)
@@ -97,25 +113,33 @@ def add_session_key(file_path, server_url, session_key, master_key):
     parser = argparse.ArgumentParser(description='Process secret and path arguments')
     parser.add_argument('srcpath', help='Path of the file to be uploaded')
     parser.add_argument('destpath', help='Path of the destination file within data folder')
-    parser.add_argument('--server', help='Name of the POD server', default='pods.test.solidcommunity.au')
     args = parser.parse_args()
 
-    server = args.server
-    server_url = f'https://{args.server}/'
-
     # Destination file path format: server_path/pod_name/app_name/data/data_file
 
     dest_path = os.path.abspath(args.destpath)
     assert dest_path.startswith(server_path)
     items = dest_path.replace(server_path, '').split('/')
     assert len(items) >= 4
+    pod_name = items[0]
+    app_name = items[1]
     assert items[2] == 'data'
-    file_path = '/'.join(items)  # pod_name/app_name/data/data_file
+    app_path = f'{server_path}{pod_name}/{app_name}'
 
     # Generate master encryption key from the user-provided security key
 
     security_key_str = getpass(prompt='Security Key: ')
     master_key = gen_master_key(security_key_str)
+    verify_key = gen_verify_key(security_key_str)
+
+    # Verify security key
+
+    _map = parse_ttl(f'{app_path}/encryption/enc-keys.ttl')
+    enc_key_url, enc_key_map = list(_map.items())[0]
+    verify_key_stored = enc_key_map[verify_key_pred]
+    if verify_key.decode('utf-8') != verify_key_stored:
+        print('ERROR: Incorrect security key (verification failed).')
+        sys.exit(0)
 
     # Read content of source file
 
@@ -124,5 +148,9 @@ def add_session_key(file_path, server_url, session_key, master_key):
     with open(src_path, 'r') as f:
         file_content = f.read()
 
-    upload_file(file_content, file_path, server_url, master_key)
+    # Write encrypted content to POD
+
+    r = urlparse(enc_key_url)
+    file_url = '/'.join([f'{r.scheme}://{r.netloc}'] + items)
+    upload_file(file_url, file_content, master_key)