added example script to transfer old data to influx
This commit is contained in:
parent
85f179bf4d
commit
303bfafcac
54
extras/import_old_redings.py
Normal file
54
extras/import_old_redings.py
Normal file
|
|
@ -0,0 +1,54 @@
|
||||||
|
|
||||||
|
import django
|
||||||
|
from time import time
|
||||||
|
from pyscada.models import Variable
|
||||||
|
from concurrent.futures import ThreadPoolExecutor, ProcessPoolExecutor
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
def subprocess_setup():
|
||||||
|
django.setup()
|
||||||
|
from django.db import connections
|
||||||
|
for conn in connections.all():
|
||||||
|
conn.close()
|
||||||
|
|
||||||
|
def process_variables(variable_ids):
|
||||||
|
for variable_id in variable_ids:
|
||||||
|
v = Variable.objects.get(pk=variable_id)
|
||||||
|
points = []
|
||||||
|
i = 0
|
||||||
|
# now days primary key offset, -> ms
|
||||||
|
min_time = (time()-(1 * 24 * 60 * 60)) * 2097152 * 1000
|
||||||
|
data = v.recordeddata_set.filter(pk__gt=min_time)
|
||||||
|
i_max = data.count()
|
||||||
|
if i_max == 0:
|
||||||
|
continue
|
||||||
|
print(f"{v} {i} {i_max} {i/i_max*100}")
|
||||||
|
if not hasattr(v.datasource,"influxdatabase"):
|
||||||
|
continue
|
||||||
|
new_point = v.datasource.influxdatabase.create_data_element_from_variable
|
||||||
|
for rc in data.iterator():
|
||||||
|
points.append(new_point(v ,rc.timestamp, rc.value()))
|
||||||
|
i += 1
|
||||||
|
if i%5000==0:
|
||||||
|
v.datasource.influxdatabase.write_points(points)
|
||||||
|
print(f"{v} {i} {i_max} {i/i_max*100:1.2f}")
|
||||||
|
points = []
|
||||||
|
v.datasource.influxdatabase.write_points(points)
|
||||||
|
print(f"{v} {i} {i_max} {i/i_max*100:1.2f}")
|
||||||
|
points = []
|
||||||
|
|
||||||
|
variables = []
|
||||||
|
workers = 1
|
||||||
|
for worker_id in range(workers):
|
||||||
|
variables.append([])
|
||||||
|
worker = 0
|
||||||
|
|
||||||
|
for em in Variable.objects.all():
|
||||||
|
variables[worker % workers].append(em.pk)
|
||||||
|
worker += 1
|
||||||
|
|
||||||
|
#with ProcessPoolExecutor(max_workers=workers, initializer=subprocess_setup) as executor:
|
||||||
|
# for worker_id in range(workers):
|
||||||
|
# executor.submit(process_variables, variables[worker_id])
|
||||||
|
process_variables(variables[0])
|
||||||
Loading…
Reference in New Issue
Block a user