how to handle missing datapoints

Phillip Weeks

Phil, I tried the code you provided and this part of the script returns the values array with 0 length {} even though I see hundreds of pointvalues on this context point between these dates. Anything obvious?

.var periodBegin = new Date(2017, 4, 1).getTime();
var periodEnd = new Date(2017, 4, 14).getTime();
var values = p.pointValuesBetween( periodBegin, periodEnd );

This is a printout of the result:

0
{
value: 228.5,
time: 1492146208215,
millis: 215,
second: 28,
minute: 3,
hour: 1,
day: 14,
dayOfWeek: 6,
dayOfYear: 104,
month: 4,
year: 2017,
last(count): PointValueTime[count],
lastValue: PointValueTime(228.5@2017/04/14 01:03:28.215),
lastValue(count): PointValueTime,
set(value): ,
set(value, timestamp): ,
pointValuesBetween(timestamp, timestamp): PointValueTime[],
pointValuesSince(timestamp): PointValueTime[],
pointValuesBefore(timestamp): PointValueTime[],
pointValuesAfter(timestamp): PointValueTime[],
pointValueAt(timestamp): PointValueTime,
ago(periodType): double,
ago(periodType, periods): double,
past(periodType): AnalogStatisticsWrapper,
past(periodType, periods): AnalogStatisticsWrapper,
prev(periodType): AnalogStatisticsWrapper,
prev(periodType, periods): AnalogStatisticsWrapper,
previous(periodType): AnalogStatisticsWrapper,
previous(periodType, periods): AnalogStatisticsWrapper,
stats(from, to): AnalogStatisticsWrapper,
}

This is the context point stats for the past 15 days:
Statistics (as of 2017/04/14 01:59:14) Time period: 15
Get statistics
Start: 2017/03/30 23:00
End: 2017/04/14 01:59
Minimum: 23.00 L @ Mar 30 23:00
Maximum: 228.50 L @ Apr 13 14:05
Average: 107.36 L
Integral: 131019642.6 L·s
Sum: 1470286.00 L
Log entries: 15264

phildunlap

It's probably because I spaced on months being indexed to 0, so 4 is May instead of April. Day of the month is indexed to 1, though, so I guess you're looking for new Date(2017, 3, 1) and new Date(2017, 3, 14)

Phillip Weeks

OK yes that was it
Great I have it inserting 5000 points for one point history now Id like to get this working..
For(i=1;i<73;i++){....
eval('var values = p'+i+'.pointValuesBetween( periodBegin, periodEnd )');

eval('p'+i+'.set((val1.value + (val2.value - val1.value) * (newDataTime - val1.time) / (val2.time - val1.time)), newDataTime )');

It complains of error
Any ideas..

phildunlap

Capital F in for?

I guess you're in the state where the script doesn't output the error message? Sometimes clearing your cache and refreshing can relieve that.

Phillip Weeks

Hi Phil .. almost done with this issue.. using var x = eval('p'+i) I can assign x = p1 thru p9 but for some reason it won't assign p10 or higher to x generates an error Any ideas why?

if (!INTERPOLATE_FLAG.value===true){
var DatapointsAdded = 0;
var LongestSpan = 0;
var minimumDataInterval = 60000; //60 seconds
var periodBegin = new Date(2017, 2, 1).getTime(); //You probably wish to set these ranges
var periodEnd = new Date(2017, 3, 14).getTime();

INTERPOLATE_FLAG.set(false,periodEnd);

for (var i = 1; i<9; i++){
var x = eval('p'+i);
print(x);
var values = x.pointValuesSince(periodBegin);
for(var k = 0; k < values.length-1; k+=1) {
var CurrentSpan =0;
var val1 = values[k];
var val2 = values[k+1];
if( val2.time - val1.time > minimumDataInterval ) {
var newDataTime = val1.time + minimumDataInterval;
while( newDataTime < val2.time ) {
x.set((val1.value + (val2.value - val1.value) * (newDataTime - val1.time) / (val2.time - val1.time)), newDataTime );
DatapointsAdded += 1;
CurrentSpan += 1;
if (CurrentSpan > LongestSpan){
LongestSpan = CurrentSpan;
}
}
}
}
}

Datapoints_Added.set(DatapointsAdded,periodEnd);
Longest_Span.set(LongestSpan,periodEnd);

}
RESULTS:

Setting point START INTERPOLATION to false @14/04/2017 00:00:00
{
value: 228.5,
time: 1492189828215,
millis: 215,
second: 28,
minute: 10,
hour: 13,
day: 14,
dayOfWeek: 6,
dayOfYear: 104,
month: 4,
year: 2017,
last(count): PointValueTime[count],
lastValue: PointValueTime(228.5@2017/04/14 13:10:28.215),
lastValue(count): PointValueTime,
set(value): ,
set(value, timestamp): ,
pointValuesBetween(timestamp, timestamp): PointValueTime[],
pointValuesSince(timestamp): PointValueTime[],
pointValuesBefore(timestamp): PointValueTime[],
pointValuesAfter(timestamp): PointValueTime[],
pointValueAt(timestamp): PointValueTime,
ago(periodType): double,
ago(periodType, periods): double,
past(periodType): AnalogStatisticsWrapper,
past(periodType, periods): AnalogStatisticsWrapper,
prev(periodType): AnalogStatisticsWrapper,
prev(periodType, periods): AnalogStatisticsWrapper,
previous(periodType): AnalogStatisticsWrapper,
previous(periodType, periods): AnalogStatisticsWrapper,
stats(from, to): AnalogStatisticsWrapper,
}

{
value: 48.0,
time: 1492189828215,
millis: 215,
second: 28,
minute: 10,
hour: 13,
day: 14,
dayOfWeek: 6,
dayOfYear: 104,
month: 4,
year: 2017,
last(count): PointValueTime[count],
lastValue: PointValueTime(48.0@2017/04/14 13:10:28.215),
lastValue(count): PointValueTime,
set(value): ,
set(value, timestamp): ,
pointValuesBetween(timestamp, timestamp): PointValueTime[],
pointValuesSince(timestamp): PointValueTime[],
pointValuesBefore(timestamp): PointValueTime[],
pointValuesAfter(timestamp): PointValueTime[],
pointValueAt(timestamp): PointValueTime,
ago(periodType): double,
ago(periodType, periods): double,
past(periodType): AnalogStatisticsWrapper,
past(periodType, periods): AnalogStatisticsWrapper,
prev(periodType): AnalogStatisticsWrapper,
prev(periodType, periods): AnalogStatisticsWrapper,
previous(periodType): AnalogStatisticsWrapper,
previous(periodType, periods): AnalogStatisticsWrapper,
stats(from, to): AnalogStatisticsWrapper,
}

{
value: 1401.0,
time: 1492189828215,
millis: 215,
second: 28,
minute: 10,
hour: 13,
day: 14,
dayOfWeek: 6,
dayOfYear: 104,
month: 4,
year: 2017,
last(count): PointValueTime[count],
lastValue: PointValueTime(1401.0@2017/04/14 13:10:28.215),
lastValue(count): PointValueTime,
set(value): ,
set(value, timestamp): ,
pointValuesBetween(timestamp, timestamp): PointValueTime[],
pointValuesSince(timestamp): PointValueTime[],
pointValuesBefore(timestamp): PointValueTime[],
pointValuesAfter(timestamp): PointValueTime[],
pointValueAt(timestamp): PointValueTime,
ago(periodType): double,
ago(periodType, periods): double,
past(periodType): AnalogStatisticsWrapper,
past(periodType, periods): AnalogStatisticsWrapper,
prev(periodType): AnalogStatisticsWrapper,
prev(periodType, periods): AnalogStatisticsWrapper,
previous(periodType): AnalogStatisticsWrapper,
previous(periodType, periods): AnalogStatisticsWrapper,
stats(from, to): AnalogStatisticsWrapper,
}

{
value: 821.5,
time: 1492189828215,
millis: 215,
second: 28,
minute: 10,
hour: 13,
day: 14,
dayOfWeek: 6,
dayOfYear: 104,
month: 4,
year: 2017,
last(count): PointValueTime[count],
lastValue: PointValueTime(821.5@2017/04/14 13:10:28.215),
lastValue(count): PointValueTime,
set(value): ,
set(value, timestamp): ,
pointValuesBetween(timestamp, timestamp): PointValueTime[],
pointValuesSince(timestamp): PointValueTime[],
pointValuesBefore(timestamp): PointValueTime[],
pointValuesAfter(timestamp): PointValueTime[],
pointValueAt(timestamp): PointValueTime,
ago(periodType): double,
ago(periodType, periods): double,
past(periodType): AnalogStatisticsWrapper,
past(periodType, periods): AnalogStatisticsWrapper,
prev(periodType): AnalogStatisticsWrapper,
prev(periodType, periods): AnalogStatisticsWrapper,
previous(periodType): AnalogStatisticsWrapper,
previous(periodType, periods): AnalogStatisticsWrapper,
stats(from, to): AnalogStatisticsWrapper,
}

{
value: 836.5,
time: 1492189828215,
millis: 215,
second: 28,
minute: 10,
hour: 13,
day: 14,
dayOfWeek: 6,
dayOfYear: 104,
month: 4,
year: 2017,
last(count): PointValueTime[count],
lastValue: PointValueTime(836.5@2017/04/14 13:10:28.215),
lastValue(count): PointValueTime,
set(value): ,
set(value, timestamp): ,
pointValuesBetween(timestamp, timestamp): PointValueTime[],
pointValuesSince(timestamp): PointValueTime[],
pointValuesBefore(timestamp): PointValueTime[],
pointValuesAfter(timestamp): PointValueTime[],
pointValueAt(timestamp): PointValueTime,
ago(periodType): double,
ago(periodType, periods): double,
past(periodType): AnalogStatisticsWrapper,
past(periodType, periods): AnalogStatisticsWrapper,
prev(periodType): AnalogStatisticsWrapper,
prev(periodType, periods): AnalogStatisticsWrapper,
previous(periodType): AnalogStatisticsWrapper,
previous(periodType, periods): AnalogStatisticsWrapper,
stats(from, to): AnalogStatisticsWrapper,
}

{
value: 250.5,
time: 1492189828215,
millis: 215,
second: 28,
minute: 10,
hour: 13,
day: 14,
dayOfWeek: 6,
dayOfYear: 104,
month: 4,
year: 2017,
last(count): PointValueTime[count],
lastValue: PointValueTime(250.5@2017/04/14 13:10:28.215),
lastValue(count): PointValueTime,
set(value): ,
set(value, timestamp): ,
pointValuesBetween(timestamp, timestamp): PointValueTime[],
pointValuesSince(timestamp): PointValueTime[],
pointValuesBefore(timestamp): PointValueTime[],
pointValuesAfter(timestamp): PointValueTime[],
pointValueAt(timestamp): PointValueTime,
ago(periodType): double,
ago(periodType, periods): double,
past(periodType): AnalogStatisticsWrapper,
past(periodType, periods): AnalogStatisticsWrapper,
prev(periodType): AnalogStatisticsWrapper,
prev(periodType, periods): AnalogStatisticsWrapper,
previous(periodType): AnalogStatisticsWrapper,
previous(periodType, periods): AnalogStatisticsWrapper,
stats(from, to): AnalogStatisticsWrapper,
}

{
value: 1005.5,
time: 1492189828215,
millis: 215,
second: 28,
minute: 10,
hour: 13,
day: 14,
dayOfWeek: 6,
dayOfYear: 104,
month: 4,
year: 2017,
last(count): PointValueTime[count],
lastValue: PointValueTime(1005.5@2017/04/14 13:10:28.215),
lastValue(count): PointValueTime,
set(value): ,
set(value, timestamp): ,
pointValuesBetween(timestamp, timestamp): PointValueTime[],
pointValuesSince(timestamp): PointValueTime[],
pointValuesBefore(timestamp): PointValueTime[],
pointValuesAfter(timestamp): PointValueTime[],
pointValueAt(timestamp): PointValueTime,
ago(periodType): double,
ago(periodType, periods): double,
past(periodType): AnalogStatisticsWrapper,
past(periodType, periods): AnalogStatisticsWrapper,
prev(periodType): AnalogStatisticsWrapper,
prev(periodType, periods): AnalogStatisticsWrapper,
previous(periodType): AnalogStatisticsWrapper,
previous(periodType, periods): AnalogStatisticsWrapper,
stats(from, to): AnalogStatisticsWrapper,
}

{
value: 0.0,
time: 1492189828215,
millis: 215,
second: 28,
minute: 10,
hour: 13,
day: 14,
dayOfWeek: 6,
dayOfYear: 104,
month: 4,
year: 2017,
last(count): PointValueTime[count],
lastValue: PointValueTime(0.0@2017/04/14 13:10:28.215),
lastValue(count): PointValueTime,
set(value): ,
set(value, timestamp): ,
pointValuesBetween(timestamp, timestamp): PointValueTime[],
pointValuesSince(timestamp): PointValueTime[],
pointValuesBefore(timestamp): PointValueTime[],
pointValuesAfter(timestamp): PointValueTime[],
pointValueAt(timestamp): PointValueTime,
ago(periodType): double,
ago(periodType, periods): double,
past(periodType): AnalogStatisticsWrapper,
past(periodType, periods): AnalogStatisticsWrapper,
prev(periodType): AnalogStatisticsWrapper,
prev(periodType, periods): AnalogStatisticsWrapper,
previous(periodType): AnalogStatisticsWrapper,
previous(periodType, periods): AnalogStatisticsWrapper,
stats(from, to): AnalogStatisticsWrapper,
}

Setting point NEW_DATAPOINTS_ADDED to 40841.0 @14/04/2017 00:00:00
Setting point Longest Datapoint Gap to 2232.0 @14/04/2017 00:00:00

phildunlap

Somewhere between anonymous functions and an eval() function there is plenty of room to make JavaScript confusing. I would avoid the eval function, personally, as its cleverness is often not worth its extra complexity or risk. But, that's up to you.

The likelihood is, if it has the problem on #10, that p10 is not a valid variable name or p10 is disabled.

phildunlap

Hi Phillip,

I just put Scripting version 1.2.1 into the store. It has a fix for the reason the error messages may not be printing beneath the script box. Could be helpful!

Phillip Weeks

Thanks Phil I'll try it out.

phildunlap

Hi Phillip,

It looks like I made some mistakes when writing the interpolate script. From testing, here's an updated version of the base functionality...

function interpolate(valueTime1, valueTime2, time) {
  //You can implement this other ways, but this is simple linear I believe
  return parseFloat(valueTime1.value) + ((valueTime2.value - valueTime1.value) * (time - valueTime1.time)) / (valueTime2.time - valueTime1.time);
}

var minimumDataInterval = 5000; //5 seconds
var periodBegin = new Date(2017, 3, 12).getTime(); //You probably wish to set these ranges
var periodEnd = new Date(2017, 3, 13).getTime();
var values = p.pointValuesBetween( periodBegin, periodEnd );
for(var k = 0; k < values.length-1; k+=1) {
  var val1 = values[k];
  var val2 = values[k+1];
  if( val2.time - val1.time > minimumDataInterval ) {
    var newDataTime = val1.time + minimumDataInterval;
    while( newDataTime < val2.time ) {
      p.set( interpolate(val1, val2, newDataTime), newDataTime );
      newDataTime += minimumDataInterval;
  }
}
}

Here's the setup from my test:

{
   "dataSources":[
      {
         "xid":"DS_3b6b1d05-2aad-456f-895b-fe891ebc10a7",
         "name":"Scripting-BaseData",
         "enabled":false,
         "type":"SCRIPTING",
         "alarmLevels":{
            "SCRIPT_ERROR":"URGENT",
            "DATA_TYPE_ERROR":"URGENT",
            "POLL_ABORTED":"URGENT",
            "LOG_ERROR":"URGENT"
         },
         "purgeType":"YEARS",
         "context":[
            {
               "varName":"p1",
               "dataPointXid":"DP_ae686073-a60f-481e-9d25-3ddc3e78ef88"
            }
         ],
         "logLevel":"NONE",
         "cronPattern":"0\/1 * * * * ?",
         "executionDelaySeconds":0,
         "historicalSetting":false,
         "script":"if(!p1.value) {\n    p1.set(0, 0);\n    p1.set(1000, 1000);\n}",
         "scriptPermissions":{
            "customPermissions":"",
            "dataPointReadPermissions":"superadmin",
            "dataPointSetPermissions":"superadmin",
            "dataSourcePermissions":"superadmin"
         },
         "editPermission":"",
         "purgeOverride":false,
         "purgePeriod":1
      },
      {
         "xid":"DS_af7afc6b-50b5-4c33-84e0-f9297514df28",
         "name":"Scripting-Interpolate",
         "enabled":false,
         "type":"SCRIPTING",
         "alarmLevels":{
            "SCRIPT_ERROR":"URGENT",
            "DATA_TYPE_ERROR":"URGENT",
            "POLL_ABORTED":"URGENT",
            "LOG_ERROR":"URGENT"
         },
         "purgeType":"YEARS",
         "context":[
            {
               "varName":"p",
               "dataPointXid":"DP_ae686073-a60f-481e-9d25-3ddc3e78ef88"
            }
         ],
         "logLevel":"NONE",
         "cronPattern":"0\/1 * * * * ?",
         "executionDelaySeconds":0,
         "historicalSetting":true,
         "script":"function interpolate(valueTime1, valueTime2, time) {\n  \/\/You can implement this other ways, but this is simple linear I believe\n  return parseFloat(valueTime1.value) + ((valueTime2.value - valueTime1.value) * (time - valueTime1.time)) \/ (valueTime2.time - valueTime1.time);\n}\nif(p.last(100).length < 5) {\nvar minimumDataInterval = 2; \/\/5 seconds\nvar periodBegin = 0; \/\/You probably wish to set these ranges\nvar periodEnd = 1001;\nvar values = p.pointValuesBetween( periodBegin, periodEnd );\nfor(var k = 0; k < values.length-1; k+=1) {\n  var val1 = values[k];\n  var val2 = values[k+1];\n  if( val2.time - val1.time > minimumDataInterval ) {\n    var newDataTime = val1.time + minimumDataInterval;\n    while( newDataTime < val2.time ) {\n      p.set( interpolate(val1, val2, newDataTime), newDataTime );\n      newDataTime += minimumDataInterval;\n  }\n}\n}}",
         "scriptPermissions":{
            "customPermissions":"",
            "dataPointReadPermissions":"superadmin",
            "dataPointSetPermissions":"superadmin",
            "dataSourcePermissions":"superadmin"
         },
         "editPermission":"",
         "purgeOverride":false,
         "purgePeriod":1
      },
      {
         "xid":"DS_d68e5a18-5457-4fe3-a76b-3fc4c1c7120b",
         "name":"vrtds",
         "enabled":true,
         "type":"VIRTUAL",
         "alarmLevels":{
            "POLL_ABORTED":"URGENT"
         },
         "purgeType":"YEARS",
         "updatePeriodType":"MINUTES",
         "polling":false,
         "updatePeriods":5,
         "editPermission":"",
         "purgeOverride":false,
         "purgePeriod":1
      }
   ],
   "dataPoints":[
      {
         "xid":"DP_ae686073-a60f-481e-9d25-3ddc3e78ef88",
         "name":"test",
         "enabled":true,
         "loggingType":"ALL",
         "intervalLoggingPeriodType":"MINUTES",
         "intervalLoggingType":"AVERAGE",
         "purgeType":"YEARS",
         "pointLocator":{
            "dataType":"NUMERIC",
            "changeType":{
               "type":"NO_CHANGE",
               "startValue":"0"
            },
            "settable":true
         },
         "eventDetectors":[
         ],
         "plotType":"SPLINE",
         "rollup":"NONE",
         "unit":"",
         "chartColour":"",
         "chartRenderer":{
            "type":"IMAGE",
            "timePeriodType":"DAYS",
            "numberOfPeriods":1
         },
         "dataSourceXid":"DS_d68e5a18-5457-4fe3-a76b-3fc4c1c7120b",
         "defaultCacheSize":1,
         "deviceName":"vrtds",
         "discardExtremeValues":false,
         "discardHighLimit":1.7976931348623157E308,
         "discardLowLimit":-1.7976931348623157E308,
         "intervalLoggingPeriod":1,
         "intervalLoggingSampleWindowSize":0,
         "overrideIntervalLoggingSamples":false,
         "preventSetExtremeValues":false,
         "purgeOverride":false,
         "purgePeriod":1,
         "readPermission":"",
         "setExtremeHighLimit":1.7976931348623157E308,
         "setExtremeLowLimit":-1.7976931348623157E308,
         "setPermission":"",
         "textRenderer":{
            "type":"ANALOG",
            "useUnitAsSuffix":true,
            "unit":"",
            "renderedUnit":"",
            "format":"0.00"
         },
         "tolerance":0.0
      }
   ]
}

Phillip Weeks

Thanks again Phil. That helped :)