JACoW is a publisher in Geneva, Switzerland that publishes the proceedings of accelerator conferences held around the world by an international collaboration of editors.
@inproceedings{werder:icalepcs2023-we3bco01,
author = {D. Werder and T. Humar},
title = {{Modular and Scalable Archiving for EPICS and Other Time Series Using ScyllaDB and Rust}},
% booktitle = {Proc. ICALEPCS'23},
booktitle = {Proc. 19th Int. Conf. Accel. Large Exp. Phys. Control Syst. (ICALEPCS'23)},
eventdate = {2023-10-09/2023-10-13},
pages = {1008--1011},
paper = {WE3BCO01},
language = {english},
keywords = {database, EPICS, FEL, MMI, operation},
venue = {Cape Town, South Africa},
series = {International Conference on Accelerator and Large Experimental Physics Control Systems},
number = {19},
publisher = {JACoW Publishing, Geneva, Switzerland},
month = {02},
year = {2024},
issn = {2226-0358},
isbn = {978-3-95450-238-7},
doi = {10.18429/JACoW-ICALEPCS2023-WE3BCO01},
url = {https://jacow.org/icalepcs2023/papers/we3bco01.pdf},
abstract = {{At PSI we currently run too many different products with the common goal of archiving timestamped data. This includes EPICS Channel Archiver as well as Archiver Appliance for EPICS IOC’s, a buffer storage for beam-synchronous data at SwissFEL, and more. This number of monolithic solutions is too large to maintain and overlaps in functionality. Each solution brings their own storage engine, file format and centralized design which is hard to scale. In this talk I report on how we factored the system into modular components with clean interfaces. At the core, the different storage engines and file formats have been replaced by ScyllaDB, which is an open source product with enterprise support and remarkable adoption in the industry. We gain from its distributed, fault-tolerant and scalable design. The ingest of data into ScyllaDB is factored into components according to the different type of protocols of the sources, e.g. Channel Access. Here we build upon the Rust language and achieve robust, maintainable and performant services. One interface to access and process the recorded data is the HTTP retrieval service. This service offers e.g. search among the channels by various criteria, full event data as well as aggregated and binned data in either json or binary formats. This service can also run user-defined data transformations and act as a source for Grafana for a first view into recorded channel data. Our setup for SwissFEL ingests the ~370k EPICS updates/s from ~220k PVs (scalar and waveform), having rates between 0.1 and 100 Hz.}},
}