'use strict'; var timeFormat = require('d3-time-format').timeFormat; var isNumeric = require('fast-isnumeric'); var Loggers = require('./loggers'); var mod = require('./mod').mod; var constants = require('../constants/numerical'); var BADNUM = constants.BADNUM; var ONEDAY = constants.ONEDAY; var ONEHOUR = constants.ONEHOUR; var ONEMIN = constants.ONEMIN; var ONESEC = constants.ONESEC; var EPOCHJD = constants.EPOCHJD; var Registry = require('../registry'); var utcFormat = require('d3-time-format').utcFormat; var DATETIME_REGEXP = /^\s*(-?\d\d\d\d|\d\d)(-(\d?\d)(-(\d?\d)([ Tt]([01]?\d|2[0-3])(:([0-5]\d)(:([0-5]\d(\.\d+)?))?(Z|z|[+\-]\d\d(:?\d\d)?)?)?)?)?)?\s*$/m; // special regex for chinese calendars to support yyyy-mmi-dd etc for intercalary months var DATETIME_REGEXP_CN = /^\s*(-?\d\d\d\d|\d\d)(-(\d?\di?)(-(\d?\d)([ Tt]([01]?\d|2[0-3])(:([0-5]\d)(:([0-5]\d(\.\d+)?))?(Z|z|[+\-]\d\d(:?\d\d)?)?)?)?)?)?\s*$/m; // for 2-digit years, the first year we map them onto var YFIRST = new Date().getFullYear() - 70; function isWorldCalendar(calendar) { return ( calendar && Registry.componentsRegistry.calendars && typeof calendar === 'string' && calendar !== 'gregorian' ); } /* * dateTick0: get the canonical tick for this calendar * * integer weekdays : Saturday: 0, Sunday: 1, Monday: 2, etc. */ exports.dateTick0 = function(calendar, dayOfWeek) { var tick0 = _dateTick0(calendar, !!dayOfWeek); if(dayOfWeek < 2) return tick0; var v = exports.dateTime2ms(tick0, calendar); v += ONEDAY * (dayOfWeek - 1); // shift Sunday to Monday, etc. return exports.ms2DateTime(v, 0, calendar); }; /* * _dateTick0: get the canonical tick for this calendar * * bool sunday is for week ticks, shift it to a Sunday. */ function _dateTick0(calendar, sunday) { if(isWorldCalendar(calendar)) { return sunday ? Registry.getComponentMethod('calendars', 'CANONICAL_SUNDAY')[calendar] : Registry.getComponentMethod('calendars', 'CANONICAL_TICK')[calendar]; } else { return sunday ? '2000-01-02' : '2000-01-01'; } } /* * dfltRange: for each calendar, give a valid default range */ exports.dfltRange = function(calendar) { if(isWorldCalendar(calendar)) { return Registry.getComponentMethod('calendars', 'DFLTRANGE')[calendar]; } else { return ['2000-01-01', '2001-01-01']; } }; // is an object a javascript date? exports.isJSDate = function(v) { return typeof v === 'object' && v !== null && typeof v.getTime === 'function'; }; // The absolute limits of our date-time system // This is a little weird: we use MIN_MS and MAX_MS in dateTime2ms // but we use dateTime2ms to calculate them (after defining it!) var MIN_MS, MAX_MS; /** * dateTime2ms - turn a date object or string s into milliseconds * (relative to 1970-01-01, per javascript standard) * optional calendar (string) to use a non-gregorian calendar * * Returns BADNUM if it doesn't find a date * * strings should have the form: * * -?YYYY-mm-ddHH:MM:SS.sss? * * : space (our normal standard) or T or t (ISO-8601) * : Z, z, [+\-]HH:?MM or [+\-]HH and we THROW IT AWAY * this format comes from https://tools.ietf.org/html/rfc3339#section-5.6 * and 4.2.5.1 Difference between local time and UTC of day (ISO-8601) * but we allow it even with a space as the separator * * May truncate after any full field, and sss can be any length * even >3 digits, though javascript dates truncate to milliseconds, * we keep as much as javascript numeric precision can hold, but we only * report back up to 100 microsecond precision, because most dates support * this precision (close to 1970 support more, very far away support less) * * Expanded to support negative years to -9999 but you must always * give 4 digits, except for 2-digit positive years which we assume are * near the present time. * Note that we follow ISO 8601:2004: there *is* a year 0, which * is 1BC/BCE, and -1===2BC etc. * * World calendars: not all of these *have* agreed extensions to this full range, * if you have another calendar system but want a date range outside its validity, * you can use a gregorian date string prefixed with 'G' or 'g'. * * Where to cut off 2-digit years between 1900s and 2000s? * from https://docs.microsoft.com/en-us/office/troubleshoot/excel/two-digit-year-numbers#the-2029-rule: * 1930-2029 (the most retro of all...) * but in my mac chrome from eg. d=new Date(Date.parse('8/19/50')): * 1950-2049 * by Java, from http://stackoverflow.com/questions/2024273/: * now-80 - now+19 * or FileMaker Pro, from * https://fmhelp.filemaker.com/help/18/fmp/en/index.html#page/FMP_Help/dates-with-two-digit-years.html: * now-70 - now+29 * but python strptime etc, via * http://docs.python.org/py3k/library/time.html: * 1969-2068 (super forward-looking, but static, not sliding!) * * lets go with now-70 to now+29, and if anyone runs into this problem * they can learn the hard way not to use 2-digit years, as no choice we * make now will cover all possibilities. mostly this will all be taken * care of in initial parsing, should only be an issue for hand-entered data * currently (2016) this range is: * 1946-2045 */ exports.dateTime2ms = function(s, calendar) { // first check if s is a date object if(exports.isJSDate(s)) { // Convert to the UTC milliseconds that give the same // hours as this date has in the local timezone var tzOffset = s.getTimezoneOffset() * ONEMIN; var offsetTweak = (s.getUTCMinutes() - s.getMinutes()) * ONEMIN + (s.getUTCSeconds() - s.getSeconds()) * ONESEC + (s.getUTCMilliseconds() - s.getMilliseconds()); if(offsetTweak) { var comb = 3 * ONEMIN; tzOffset = tzOffset - comb / 2 + mod(offsetTweak - tzOffset + comb / 2, comb); } s = Number(s) - tzOffset; if(s >= MIN_MS && s <= MAX_MS) return s; return BADNUM; } // otherwise only accept strings and numbers if(typeof s !== 'string' && typeof s !== 'number') return BADNUM; s = String(s); var isWorld = isWorldCalendar(calendar); // to handle out-of-range dates in international calendars, accept // 'G' as a prefix to force the built-in gregorian calendar. var s0 = s.charAt(0); if(isWorld && (s0 === 'G' || s0 === 'g')) { s = s.substr(1); calendar = ''; } var isChinese = isWorld && calendar.substr(0, 7) === 'chinese'; var match = s.match(isChinese ? DATETIME_REGEXP_CN : DATETIME_REGEXP); if(!match) return BADNUM; var y = match[1]; var m = match[3] || '1'; var d = Number(match[5] || 1); var H = Number(match[7] || 0); var M = Number(match[9] || 0); var S = Number(match[11] || 0); if(isWorld) { // disallow 2-digit years for world calendars if(y.length === 2) return BADNUM; y = Number(y); var cDate; try { var calInstance = Registry.getComponentMethod('calendars', 'getCal')(calendar); if(isChinese) { var isIntercalary = m.charAt(m.length - 1) === 'i'; m = parseInt(m, 10); cDate = calInstance.newDate(y, calInstance.toMonthIndex(y, m, isIntercalary), d); } else { cDate = calInstance.newDate(y, Number(m), d); } } catch(e) { return BADNUM; } // Invalid ... date if(!cDate) return BADNUM; return ((cDate.toJD() - EPOCHJD) * ONEDAY) + (H * ONEHOUR) + (M * ONEMIN) + (S * ONESEC); } if(y.length === 2) { y = (Number(y) + 2000 - YFIRST) % 100 + YFIRST; } else y = Number(y); // new Date uses months from 0; subtract 1 here just so we // don't have to do it again during the validity test below m -= 1; // javascript takes new Date(0..99,m,d) to mean 1900-1999, so // to support years 0-99 we need to use setFullYear explicitly // Note that 2000 is a leap year. var date = new Date(Date.UTC(2000, m, d, H, M)); date.setUTCFullYear(y); if(date.getUTCMonth() !== m) return BADNUM; if(date.getUTCDate() !== d) return BADNUM; return date.getTime() + S * ONESEC; }; MIN_MS = exports.MIN_MS = exports.dateTime2ms('-9999'); MAX_MS = exports.MAX_MS = exports.dateTime2ms('9999-12-31 23:59:59.9999'); // is string s a date? (see above) exports.isDateTime = function(s, calendar) { return (exports.dateTime2ms(s, calendar) !== BADNUM); }; // pad a number with zeroes, to given # of digits before the decimal point function lpad(val, digits) { return String(val + Math.pow(10, digits)).substr(1); } /** * Turn ms into string of the form YYYY-mm-dd HH:MM:SS.ssss * Crop any trailing zeros in time, except never stop right after hours * (we could choose to crop '-01' from date too but for now we always * show the whole date) * Optional range r is the data range that applies, also in ms. * If rng is big, the later parts of time will be omitted */ var NINETYDAYS = 90 * ONEDAY; var THREEHOURS = 3 * ONEHOUR; var FIVEMIN = 5 * ONEMIN; exports.ms2DateTime = function(ms, r, calendar) { if(typeof ms !== 'number' || !(ms >= MIN_MS && ms <= MAX_MS)) return BADNUM; if(!r) r = 0; var msecTenths = Math.floor(mod(ms + 0.05, 1) * 10); var msRounded = Math.round(ms - msecTenths / 10); var dateStr, h, m, s, msec10, d; if(isWorldCalendar(calendar)) { var dateJD = Math.floor(msRounded / ONEDAY) + EPOCHJD; var timeMs = Math.floor(mod(ms, ONEDAY)); try { dateStr = Registry.getComponentMethod('calendars', 'getCal')(calendar) .fromJD(dateJD).formatDate('yyyy-mm-dd'); } catch(e) { // invalid date in this calendar - fall back to Gyyyy-mm-dd dateStr = utcFormat('G%Y-%m-%d')(new Date(msRounded)); } // yyyy does NOT guarantee 4-digit years. YYYY mostly does, but does // other things for a few calendars, so we can't trust it. Just pad // it manually (after the '-' if there is one) if(dateStr.charAt(0) === '-') { while(dateStr.length < 11) dateStr = '-0' + dateStr.substr(1); } else { while(dateStr.length < 10) dateStr = '0' + dateStr; } // TODO: if this is faster, we could use this block for extracting // the time components of regular gregorian too h = (r < NINETYDAYS) ? Math.floor(timeMs / ONEHOUR) : 0; m = (r < NINETYDAYS) ? Math.floor((timeMs % ONEHOUR) / ONEMIN) : 0; s = (r < THREEHOURS) ? Math.floor((timeMs % ONEMIN) / ONESEC) : 0; msec10 = (r < FIVEMIN) ? (timeMs % ONESEC) * 10 + msecTenths : 0; } else { d = new Date(msRounded); dateStr = utcFormat('%Y-%m-%d')(d); // <90 days: add hours and minutes - never *only* add hours h = (r < NINETYDAYS) ? d.getUTCHours() : 0; m = (r < NINETYDAYS) ? d.getUTCMinutes() : 0; // <3 hours: add seconds s = (r < THREEHOURS) ? d.getUTCSeconds() : 0; // <5 minutes: add ms (plus one extra digit, this is msec*10) msec10 = (r < FIVEMIN) ? d.getUTCMilliseconds() * 10 + msecTenths : 0; } return includeTime(dateStr, h, m, s, msec10); }; // For converting old-style milliseconds to date strings, // we use the local timezone rather than UTC like we use // everywhere else, both for backward compatibility and // because that's how people mostly use javasript date objects. // Clip one extra day off our date range though so we can't get // thrown beyond the range by the timezone shift. exports.ms2DateTimeLocal = function(ms) { if(!(ms >= MIN_MS + ONEDAY && ms <= MAX_MS - ONEDAY)) return BADNUM; var msecTenths = Math.floor(mod(ms + 0.05, 1) * 10); var d = new Date(Math.round(ms - msecTenths / 10)); var dateStr = timeFormat('%Y-%m-%d')(d); var h = d.getHours(); var m = d.getMinutes(); var s = d.getSeconds(); var msec10 = d.getUTCMilliseconds() * 10 + msecTenths; return includeTime(dateStr, h, m, s, msec10); }; function includeTime(dateStr, h, m, s, msec10) { // include each part that has nonzero data in or after it if(h || m || s || msec10) { dateStr += ' ' + lpad(h, 2) + ':' + lpad(m, 2); if(s || msec10) { dateStr += ':' + lpad(s, 2); if(msec10) { var digits = 4; while(msec10 % 10 === 0) { digits -= 1; msec10 /= 10; } dateStr += '.' + lpad(msec10, digits); } } } return dateStr; } // normalize date format to date string, in case it starts as // a Date object or milliseconds // optional dflt is the return value if cleaning fails exports.cleanDate = function(v, dflt, calendar) { // let us use cleanDate to provide a missing default without an error if(v === BADNUM) return dflt; if(exports.isJSDate(v) || (typeof v === 'number' && isFinite(v))) { // do not allow milliseconds (old) or jsdate objects (inherently // described as gregorian dates) with world calendars if(isWorldCalendar(calendar)) { Loggers.error('JS Dates and milliseconds are incompatible with world calendars', v); return dflt; } // NOTE: if someone puts in a year as a number rather than a string, // this will mistakenly convert it thinking it's milliseconds from 1970 // that is: '2012' -> Jan. 1, 2012, but 2012 -> 2012 epoch milliseconds v = exports.ms2DateTimeLocal(+v); if(!v && dflt !== undefined) return dflt; } else if(!exports.isDateTime(v, calendar)) { Loggers.error('unrecognized date', v); return dflt; } return v; }; /* * Date formatting for ticks and hovertext */ /* * modDateFormat: Support world calendars, and add two items to * d3's vocabulary: * %{n}f where n is the max number of digits of fractional seconds * %h formats: half of the year as a decimal number [1,2] */ var fracMatch = /%\d?f/g; var halfYearMatch = /%h/g; var quarterToHalfYear = { 1: '1', 2: '1', 3: '2', 4: '2', }; function modDateFormat(fmt, x, formatter, calendar) { fmt = fmt.replace(fracMatch, function(match) { var digits = Math.min(+(match.charAt(1)) || 6, 6); var fracSecs = ((x / 1000 % 1) + 2) .toFixed(digits) .substr(2).replace(/0+$/, '') || '0'; return fracSecs; }); var d = new Date(Math.floor(x + 0.05)); fmt = fmt.replace(halfYearMatch, function() { return quarterToHalfYear[formatter('%q')(d)]; }); if(isWorldCalendar(calendar)) { try { fmt = Registry.getComponentMethod('calendars', 'worldCalFmt')(fmt, x, calendar); } catch(e) { return 'Invalid'; } } return formatter(fmt)(d); } /* * formatTime: create a time string from: * x: milliseconds * tr: tickround ('M', 'S', or # digits) * only supports UTC times (where every day is 24 hours and 0 is at midnight) */ var MAXSECONDS = [59, 59.9, 59.99, 59.999, 59.9999]; function formatTime(x, tr) { var timePart = mod(x + 0.05, ONEDAY); var timeStr = lpad(Math.floor(timePart / ONEHOUR), 2) + ':' + lpad(mod(Math.floor(timePart / ONEMIN), 60), 2); if(tr !== 'M') { if(!isNumeric(tr)) tr = 0; // should only be 'S' /* * this is a weird one - and shouldn't come up unless people * monkey with tick0 in weird ways, but we need to do something! * IN PARTICULAR we had better not display garbage (see below) * for numbers we always round to the nearest increment of the * precision we're showing, and this seems like the right way to * handle seconds and milliseconds, as they have a decimal point * and people will interpret that to mean rounding like numbers. * but for larger increments we floor the value: it's always * 2013 until the ball drops on the new year. We could argue about * which field it is where we start rounding (should 12:08:59 * round to 12:09 if we're stopping at minutes?) but for now I'll * say we round seconds but floor everything else. BUT that means * we need to never round up to 60 seconds, ie 23:59:60 */ var sec = Math.min(mod(x / ONESEC, 60), MAXSECONDS[tr]); var secStr = (100 + sec).toFixed(tr).substr(1); if(tr > 0) { secStr = secStr.replace(/0+$/, '').replace(/[\.]$/, ''); } timeStr += ':' + secStr; } return timeStr; } /* * formatDate: turn a date into tick or hover label text. * * x: milliseconds, the value to convert * fmt: optional, an explicit format string (d3 format, even for world calendars) * tr: tickround ('y', 'm', 'd', 'M', 'S', or # digits) * used if no explicit fmt is provided * formatter: locale-aware d3 date formatter for standard gregorian calendars * should be the result of exports.getD3DateFormat(gd) * calendar: optional string, the world calendar system to use * * returns the date/time as a string, potentially with the leading portion * on a separate line (after '\n') * Note that this means if you provide an explicit format which includes '\n' * the axis may choose to strip things after it when they don't change from * one tick to the next (as it does with automatic formatting) */ exports.formatDate = function(x, fmt, tr, formatter, calendar, extraFormat) { calendar = isWorldCalendar(calendar) && calendar; if(!fmt) { if(tr === 'y') fmt = extraFormat.year; else if(tr === 'm') fmt = extraFormat.month; else if(tr === 'd') { fmt = extraFormat.dayMonth + '\n' + extraFormat.year; } else { return formatTime(x, tr) + '\n' + modDateFormat(extraFormat.dayMonthYear, x, formatter, calendar); } } return modDateFormat(fmt, x, formatter, calendar); }; /* * incrementMonth: make a new milliseconds value from the given one, * having changed the month * * special case for world calendars: multiples of 12 are treated as years, * even for calendar systems that don't have (always or ever) 12 months/year * TODO: perhaps we need a different code for year increments to support this? * * ms (number): the initial millisecond value * dMonth (int): the (signed) number of months to shift * calendar (string): the calendar system to use * * changing month does not (and CANNOT) always preserve day, since * months have different lengths. The worst example of this is: * d = new Date(1970,0,31); d.setMonth(1) -> Feb 31 turns into Mar 3 * * But we want to be able to iterate over the last day of each month, * regardless of what its number is. * So shift 3 days forward, THEN set the new month, then unshift: * 1/31 -> 2/28 (or 29) -> 3/31 -> 4/30 -> ... * * Note that odd behavior still exists if you start from the 26th-28th: * 1/28 -> 2/28 -> 3/31 * but at least you can't shift any dates into the wrong month, * and ticks on these days incrementing by month would be very unusual */ var THREEDAYS = 3 * ONEDAY; exports.incrementMonth = function(ms, dMonth, calendar) { calendar = isWorldCalendar(calendar) && calendar; // pull time out and operate on pure dates, then add time back at the end // this gives maximum precision - not that we *normally* care if we're // incrementing by month, but better to be safe! var timeMs = mod(ms, ONEDAY); ms = Math.round(ms - timeMs); if(calendar) { try { var dateJD = Math.round(ms / ONEDAY) + EPOCHJD; var calInstance = Registry.getComponentMethod('calendars', 'getCal')(calendar); var cDate = calInstance.fromJD(dateJD); if(dMonth % 12) calInstance.add(cDate, dMonth, 'm'); else calInstance.add(cDate, dMonth / 12, 'y'); return (cDate.toJD() - EPOCHJD) * ONEDAY + timeMs; } catch(e) { Loggers.error('invalid ms ' + ms + ' in calendar ' + calendar); // then keep going in gregorian even though the result will be 'Invalid' } } var y = new Date(ms + THREEDAYS); return y.setUTCMonth(y.getUTCMonth() + dMonth) + timeMs - THREEDAYS; }; /* * findExactDates: what fraction of data is exact days, months, or years? * * data: array of millisecond values * calendar (string) the calendar to test against */ exports.findExactDates = function(data, calendar) { var exactYears = 0; var exactMonths = 0; var exactDays = 0; var blankCount = 0; var d; var di; var calInstance = ( isWorldCalendar(calendar) && Registry.getComponentMethod('calendars', 'getCal')(calendar) ); for(var i = 0; i < data.length; i++) { di = data[i]; // not date data at all if(!isNumeric(di)) { blankCount ++; continue; } // not an exact date if(di % ONEDAY) continue; if(calInstance) { try { d = calInstance.fromJD(di / ONEDAY + EPOCHJD); if(d.day() === 1) { if(d.month() === 1) exactYears++; else exactMonths++; } else exactDays++; } catch(e) { // invalid date in this calendar - ignore it here. } } else { d = new Date(di); if(d.getUTCDate() === 1) { if(d.getUTCMonth() === 0) exactYears++; else exactMonths++; } else exactDays++; } } exactMonths += exactYears; exactDays += exactMonths; var dataCount = data.length - blankCount; return { exactYears: exactYears / dataCount, exactMonths: exactMonths / dataCount, exactDays: exactDays / dataCount }; };