public class EtlMetricsTracker
extends Object
implements Serializable
Constructor and Description |
---|
EtlMetricsTracker() |
Modifier and Type | Method and Description |
---|---|
void |
finishTrackingExtract(String name,
DataSet dataSet,
String exception)
Finish tracking extract.
|
void |
finishTrackingLoad(String name,
Destination destination,
String exception)
Finish tracking load.
|
Long |
getCurrentNumberOfExtractedRecords(String name)
Gets the current number of extracted records.
|
Long |
getCurrentNumberOfLoadedRecords(String name)
Gets the current number of loaded records.
|
Map<String,FileMetrics> |
getFileMetrics()
Gets the file metrics.
|
String |
getFileMetricsFileName()
Gets the file metrics file name.
|
String |
getFileName()
Gets the file name.
|
String |
getFormat()
Gets the format.
|
Object |
getHighWatermark(EtlConfig config,
Scenario scenario,
Source source,
Destination destination,
Boolean... onLoad)
Gets the high watermark value.
|
Object |
getHighWatermark(String name,
Boolean... onLoad)
Gets the high watermark value.
|
Map<String,EtlMetrics> |
getMetrics()
Gets the ETL metrics.
|
TypedKeyValue<String,String> |
prepareTrackingExtract(Scenario scenario,
Source source,
Destination destination,
String sql,
EtlConfig config)
Prepare tracking extract.
|
void |
read()
Read metrics from the data folder using given format.
|
void |
readMetricsFromCsv()
Read metrics from csv.
|
void |
readMetricsFromJson()
Read metrics from json.
|
boolean |
setCurrentNumberOfExtractedRecords(String name,
Long value)
Sets the current number of extracted records.
|
boolean |
setCurrentNumberOfLoadedRecords(String name,
Long value)
Sets the current number of loaded records.
|
void |
setFileName(String fileName)
Sets the file name.
|
void |
setFormat(String format)
Sets the format.
|
void |
setLastCreatedFile(String name,
String fileName)
Sets the last created file.
|
void |
setLastReadFile(String name,
String fileName)
Sets the last created file.
|
void |
startTrackingExtract(String name,
Source source,
DataSet dataSet,
Destination destination,
String flow)
Start tracking extract.
|
String |
startTrackingFile(FileMetrics fileMetric)
Start tracking file.
|
void |
startTrackingLoad(String name,
Destination destination,
String flow)
Start tracking load.
|
void |
stopTrackingFile(FileMetrics fileMetric,
String exception,
String action)
Stop tracking file.
|
void |
updateEtlMetrics(Map<String,EtlMetrics> from)
Update etl metrics.
|
void |
write(boolean merge)
Write metrics to the data folder.
|
void |
writeFileMetrics()
Write file metrics to the data folder.
|
void |
writeMetricsToCsv(Map<String,BaseMetrics> metrics,
String metricFileName)
Write metrics to csv file.
|
void |
writeMetricsToJson(Map<String,BaseMetrics> metric,
String metricFileName)
Write metrics to csv file.
|
public void finishTrackingExtract(String name, DataSet dataSet, String exception)
name
- the namedataSet
- the data setexception
- the exceptionpublic void finishTrackingLoad(String name, Destination destination, String exception)
name
- the namedestination
- the destinationexception
- the exceptionpublic Long getCurrentNumberOfExtractedRecords(String name)
name
- the namepublic Long getCurrentNumberOfLoadedRecords(String name)
name
- the namepublic Map<String,FileMetrics> getFileMetrics()
public String getFileMetricsFileName()
public String getFileName()
public String getFormat()
public Map<String,EtlMetrics> getMetrics()
public void updateEtlMetrics(Map<String,EtlMetrics> from)
from
- the map of etl metrics to update frompublic Object getHighWatermark(String name, Boolean... onLoad)
name
- the name of the etl blockonLoad
- if true or null - get high watermark from the onload event,
otherwise - from the onextract.public Object getHighWatermark(EtlConfig config, Scenario scenario, Source source, Destination destination, Boolean... onLoad) throws Exception
config
- the configscenario
- the scenariosource
- the sourcedestination
- the destinationonLoad
- the on loadException
- the exceptionpublic void read() throws Exception
Exception
- the exceptionpublic void readMetricsFromCsv() throws Exception
Exception
- the exceptionpublic void readMetricsFromJson() throws Exception
Exception
- the exceptionpublic boolean setCurrentNumberOfExtractedRecords(String name, Long value)
name
- the namevalue
- the valuepublic boolean setCurrentNumberOfLoadedRecords(String name, Long value)
name
- the namevalue
- the valuepublic void setFileName(String fileName)
fileName
- the fileName to setpublic void setFormat(String format)
format
- the new formatpublic void setLastCreatedFile(String name, String fileName)
name
- the namefileName
- the file namepublic void setLastReadFile(String name, String fileName)
name
- the namefileName
- the file namepublic void startTrackingExtract(String name, Source source, DataSet dataSet, Destination destination, String flow)
name
- the dataset namesource
- the sourcedataSet
- the data setdestination
- the optional destinationflow
- the flow namepublic TypedKeyValue<String,String> prepareTrackingExtract(Scenario scenario, Source source, Destination destination, String sql, EtlConfig config) throws Exception
scenario
- the scenariosource
- the sourcedestination
- the destinationsql
- the sqlconfig
- the configException
- the exceptionpublic String startTrackingFile(FileMetrics fileMetric)
fileMetric
- the file metricpublic void startTrackingLoad(String name, Destination destination, String flow)
name
- the namedestination
- the destinationflow
- the flow namepublic void stopTrackingFile(FileMetrics fileMetric, String exception, String action)
fileMetric
- the file metricexception
- the exceptionaction
- the actionpublic void write(boolean merge) throws Exception
merge
- if true system will merge metrics with existing (granted file
does exist)Exception
- the exceptionpublic void writeFileMetrics() throws Exception
Exception
- the exceptionpublic void writeMetricsToCsv(Map<String,BaseMetrics> metrics, String metricFileName) throws Exception
metrics
- the metricsmetricFileName
- the metric file nameException
- the exceptionpublic void writeMetricsToJson(Map<String,BaseMetrics> metric, String metricFileName) throws Exception
metric
- the metricmetricFileName
- the metric file nameException
- the exceptionCopyright © 2010-2020 Toolsverse. All Rights Reserved.