|
| 1 | +""" |
| 2 | +How to use RxPY to prepare batches for synchronous write into InfluxDB |
| 3 | +""" |
| 4 | + |
| 5 | +from csv import DictReader |
| 6 | + |
| 7 | +import rx |
| 8 | +from rx import operators as ops |
| 9 | + |
| 10 | +from influxdb_client import InfluxDBClient, Point |
| 11 | +from influxdb_client.client.write.retry import WritesRetry |
| 12 | +from influxdb_client.client.write_api import SYNCHRONOUS |
| 13 | + |
| 14 | + |
| 15 | +def csv_to_generator(csv_file_path): |
| 16 | + """ |
| 17 | + Parse your CSV file into generator |
| 18 | + """ |
| 19 | + for row in DictReader(open(csv_file_path, 'r')): |
| 20 | + point = Point('financial-analysis') \ |
| 21 | + .tag('type', 'vix-daily') \ |
| 22 | + .field('open', float(row['VIX Open'])) \ |
| 23 | + .field('high', float(row['VIX High'])) \ |
| 24 | + .field('low', float(row['VIX Low'])) \ |
| 25 | + .field('close', float(row['VIX Close'])) \ |
| 26 | + .time(row['Date']) |
| 27 | + yield point |
| 28 | + |
| 29 | + |
| 30 | +""" |
| 31 | +Define Retry strategy - 3 attempts => 2, 4, 8 |
| 32 | +""" |
| 33 | +retries = WritesRetry(total=3, backoff_factor=1, exponential_base=2) |
| 34 | +client = InfluxDBClient(url='http://localhost:8086', token='my-token', org='my-org', retries=retries) |
| 35 | + |
| 36 | +""" |
| 37 | +Use synchronous version of WriteApi to strongly depends on result of write |
| 38 | +""" |
| 39 | +write_api = client.write_api(write_options=SYNCHRONOUS) |
| 40 | + |
| 41 | +""" |
| 42 | +Prepare batches from generator |
| 43 | +""" |
| 44 | +batches = rx \ |
| 45 | + .from_iterable(csv_to_generator('vix-daily.csv')) \ |
| 46 | + .pipe(ops.buffer_with_count(500)) |
| 47 | + |
| 48 | + |
| 49 | +def write_batch(batch): |
| 50 | + """ |
| 51 | + Synchronous write |
| 52 | + """ |
| 53 | + print(f'Writing... {len(batch)}') |
| 54 | + write_api.write(bucket='my-bucket', record=batch) |
| 55 | + |
| 56 | + |
| 57 | +""" |
| 58 | +Write batches |
| 59 | +""" |
| 60 | +batches.subscribe(on_next=lambda batch: write_batch(batch), |
| 61 | + on_error=lambda ex: print(f'Unexpected error: {ex}'), |
| 62 | + on_completed=lambda: print('Import finished!')) |
| 63 | + |
| 64 | +""" |
| 65 | +Dispose client |
| 66 | +""" |
| 67 | +write_api.close() |
| 68 | +client.close() |
0 commit comments