diff --git a/src/main/java/org/mtransit/android/commons/provider/gtfs/GTFSProviderDBHelperUtils.kt b/src/main/java/org/mtransit/android/commons/provider/gtfs/GTFSProviderDBHelperUtils.kt index b2764acf..5ae3f573 100644 --- a/src/main/java/org/mtransit/android/commons/provider/gtfs/GTFSProviderDBHelperUtils.kt +++ b/src/main/java/org/mtransit/android/commons/provider/gtfs/GTFSProviderDBHelperUtils.kt @@ -5,6 +5,7 @@ import android.database.sqlite.SQLiteDatabase import androidx.core.database.sqlite.transaction import org.mtransit.android.commons.FileUtils import org.mtransit.android.commons.MTLog +import org.mtransit.android.commons.provider.gtfs.GTFSProviderDbHelper.DB_NAME import org.mtransit.commons.FeatureFlags import org.mtransit.commons.GTFSCommons import org.mtransit.commons.sql.SQLUtils @@ -67,7 +68,7 @@ object GTFSProviderDBHelperUtils : MTLog.Loggable { transaction { execSQL(sqlDrop) // drop if exists execSQL(sqlCreate) // create if not exists - for (file in files) { + files.forEach { file -> try { openRawResource(file).use { inputStream -> InputStreamReader(inputStream, FileUtils.getUTF8()).use { inputStreamReader -> @@ -87,11 +88,7 @@ object GTFSProviderDBHelperUtils : MTLog.Loggable { try { execSQL(sql) } catch (e: Exception) { - MTLog.w( - this, - e, - "ERROR while executing '$sql' on database '${GTFSProviderDbHelper.DB_NAME}' table '$table' file '$file'!" - ) + MTLog.w(this, e, "ERROR while executing '$sql' on database '$DB_NAME' table '$table' file '$file'!") throw e } } @@ -99,14 +96,14 @@ object GTFSProviderDBHelperUtils : MTLog.Loggable { } } } catch (e: Exception) { - MTLog.w(this, e, "ERROR while copying the database '${GTFSProviderDbHelper.DB_NAME}' table '$table' file '$file'!") + MTLog.w(this, e, "ERROR while copying the database '$DB_NAME' table '$table' file '$file'!") return false } } } return true } catch (e: Exception) { - MTLog.w(this, e, "ERROR while copying the database '${GTFSProviderDbHelper.DB_NAME}' table '$table' file!") + MTLog.w(this, e, "ERROR while copying the database '$DB_NAME' table '$table' file!") return false } } diff --git a/src/main/java/org/mtransit/android/commons/provider/gtfs/GTFSProviderDbHelper.java b/src/main/java/org/mtransit/android/commons/provider/gtfs/GTFSProviderDbHelper.java index fc5554e2..d3bd6562 100644 --- a/src/main/java/org/mtransit/android/commons/provider/gtfs/GTFSProviderDbHelper.java +++ b/src/main/java/org/mtransit/android/commons/provider/gtfs/GTFSProviderDbHelper.java @@ -189,6 +189,7 @@ private void initAllDbTables(@NonNull SQLiteDatabase db, boolean upgrade) { MTLog.i(this, "Data: deploying DB..."); final int nId = TimeUtils.currentTimeSec(); final long startInMs = TimeUtils.currentTimeMillis(); + long stepStartMs; int nbTotalOperations = 7; if (FeatureFlags.F_EXPORT_TRIP_ID) nbTotalOperations++; if (FeatureFlags.F_EXPORT_SERVICE_ID_INTS) nbTotalOperations++; @@ -212,47 +213,57 @@ private void initAllDbTables(@NonNull SQLiteDatabase db, boolean upgrade) { if (notifEnabled) NotificationUtils.setProgressAndNotify(nm, nb, nId, nbTotalOperations, ++progress); final Map allStrings = new HashMap<>(); if (FeatureFlags.F_EXPORT_STRINGS || FeatureFlags.F_EXPORT_SCHEDULE_STRINGS) { + stepStartMs = TimeUtils.currentTimeMillis(); initDbTableWithRetry(context, db, T_STRINGS, T_STRINGS_SQL_CREATE, T_STRINGS_SQL_INSERT, T_STRINGS_SQL_DROP, getStringsFiles(), 0, 0, null, null, (id, string) -> { allStrings.put(id, string); return kotlin.Unit.INSTANCE; } ); // 1st - if (Constants.DEBUG) MTLog.d(this, "Data: deploying DB... %s done (%s)", T_STRINGS, MTLog.formatDuration(TimeUtils.currentTimeMillis() - startInMs)); + if (Constants.DEBUG) MTLog.d(this, "Data: deploying DB... '%s' done (%s)", T_STRINGS, MTLog.formatDuration(TimeUtils.currentTimeMillis() - stepStartMs)); } if (notifEnabled) NotificationUtils.setProgressAndNotify(nm, nb, nId, nbTotalOperations, ++progress); + stepStartMs = TimeUtils.currentTimeMillis(); initDbTableWithRetry(context, db, T_ROUTE, T_ROUTE_SQL_CREATE, T_ROUTE_SQL_INSERT, T_ROUTE_SQL_DROP, getRouteFiles(), 0, 0, allStrings, T_ROUTE_STRINGS_COLUMN_IDX); - if (Constants.DEBUG) MTLog.d(this, "Data: deploying DB... %s done (%s)", T_ROUTE, MTLog.formatDuration(TimeUtils.currentTimeMillis() - startInMs)); + if (Constants.DEBUG) MTLog.d(this, "Data: deploying DB... '%s' done (%s)", T_ROUTE, MTLog.formatDuration(TimeUtils.currentTimeMillis() - stepStartMs)); if (notifEnabled) NotificationUtils.setProgressAndNotify(nm, nb, nId, nbTotalOperations, ++progress); + stepStartMs = TimeUtils.currentTimeMillis(); initDbTableWithRetry(context, db, T_DIRECTION, T_DIRECTION_SQL_CREATE, T_DIRECTION_SQL_INSERT, T_DIRECTION_SQL_DROP, getDirectionFiles(), 0, 0, allStrings, T_DIRECTION_STRINGS_COLUMN_IDX); - if (Constants.DEBUG) MTLog.d(this, "Data: deploying DB... %s done (%s)", T_DIRECTION, MTLog.formatDuration(TimeUtils.currentTimeMillis() - startInMs)); + if (Constants.DEBUG) MTLog.d(this, "Data: deploying DB... '%s' done (%s)", T_DIRECTION, MTLog.formatDuration(TimeUtils.currentTimeMillis() - stepStartMs)); if (notifEnabled) NotificationUtils.setProgressAndNotify(nm, nb, nId, nbTotalOperations, ++progress); + stepStartMs = TimeUtils.currentTimeMillis(); initDbTableWithRetry(context, db, T_STOP, T_STOP_SQL_CREATE, T_STOP_SQL_INSERT, T_STOP_SQL_DROP, getStopFiles(), 0, 0, allStrings, T_STOP_STRINGS_COLUMN_IDX); - if (Constants.DEBUG) MTLog.d(this, "Data: deploying DB... %s done (%s)", T_STOP, MTLog.formatDuration(TimeUtils.currentTimeMillis() - startInMs)); + if (Constants.DEBUG) MTLog.d(this, "Data: deploying DB... '%s' done (%s)", T_STOP, MTLog.formatDuration(TimeUtils.currentTimeMillis() - stepStartMs)); if (notifEnabled) NotificationUtils.setProgressAndNotify(nm, nb, nId, nbTotalOperations, ++progress); + stepStartMs = TimeUtils.currentTimeMillis(); initDbTableWithRetry(context, db, T_DIRECTION_STOPS, T_DIRECTION_STOPS_SQL_CREATE, T_DIRECTION_STOPS_SQL_INSERT, T_DIRECTION_STOPS_SQL_DROP, getDirectionStopsFiles()); - if (Constants.DEBUG) MTLog.d(this, "Data: deploying DB... %s done (%s)", T_DIRECTION_STOPS, MTLog.formatDuration(TimeUtils.currentTimeMillis() - startInMs)); + if (Constants.DEBUG) MTLog.d(this, "Data: deploying DB... '%s' done (%s)", T_DIRECTION_STOPS, MTLog.formatDuration(TimeUtils.currentTimeMillis() - stepStartMs)); if (FeatureFlags.F_EXPORT_TRIP_ID) { if (notifEnabled) NotificationUtils.setProgressAndNotify(nm, nb, nId, nbTotalOperations, ++progress); + stepStartMs = TimeUtils.currentTimeMillis(); initDbTableWithRetry(context, db, T_TRIP, T_TRIP_SQL_CREATE, T_TRIP_SQL_INSERT, T_TRIP_SQL_DROP, getTripFiles(), T_TRIP_SAME_COLUMNS_COUNT, T_TRIP_OTHER_COLUMNS_COUNT); - if (Constants.DEBUG) MTLog.d(this, "Data: deploying DB... %s done (%s)", T_TRIP, MTLog.formatDuration(TimeUtils.currentTimeMillis() - startInMs)); + if (Constants.DEBUG) MTLog.d(this, "Data: deploying DB... '%s' done (%s)", T_TRIP, MTLog.formatDuration(TimeUtils.currentTimeMillis() - stepStartMs)); } if (FeatureFlags.F_EXPORT_SERVICE_ID_INTS) { if (notifEnabled) NotificationUtils.setProgressAndNotify(nm, nb, nId, nbTotalOperations, ++progress); + stepStartMs = TimeUtils.currentTimeMillis(); initDbTableWithRetry(context, db, T_SERVICE_IDS, T_SERVICE_IDS_SQL_CREATE, T_SERVICE_IDS_SQL_INSERT, T_SERVICE_IDS_SQL_DROP, getServiceIdsFiles()); - if (Constants.DEBUG) MTLog.d(this, "Data: deploying DB... %s done (%s)", T_SERVICE_IDS, MTLog.formatDuration(TimeUtils.currentTimeMillis() - startInMs)); + if (Constants.DEBUG) MTLog.d(this, "Data: deploying DB... '%s' done (%s)", T_SERVICE_IDS, MTLog.formatDuration(TimeUtils.currentTimeMillis() - stepStartMs)); } if (notifEnabled) NotificationUtils.setProgressAndNotify(nm, nb, nId, nbTotalOperations, ++progress); + stepStartMs = TimeUtils.currentTimeMillis(); initDbTableWithRetry(context, db, T_SERVICE_DATES, T_SERVICE_DATES_SQL_CREATE, T_SERVICE_DATES_SQL_INSERT, T_SERVICE_DATES_SQL_DROP, getServiceDatesFiles(), T_SERVICE_DATES_SAME_COLUMNS_COUNT, T_SERVICE_DATES_OTHER_COLUMNS_COUNT); - if (Constants.DEBUG) MTLog.d(this, "Data: deploying DB... %s done (%s)", T_SERVICE_DATES, MTLog.formatDuration(TimeUtils.currentTimeMillis() - startInMs)); + if (Constants.DEBUG) MTLog.d(this, "Data: deploying DB... '%s' done (%s)", T_SERVICE_DATES, MTLog.formatDuration(TimeUtils.currentTimeMillis() - stepStartMs)); if (FeatureFlags.F_EXPORT_TRIP_ID_INTS) { if (notifEnabled) NotificationUtils.setProgressAndNotify(nm, nb, nId, nbTotalOperations, ++progress); + stepStartMs = TimeUtils.currentTimeMillis(); initDbTableWithRetry(context, db, T_TRIP_IDS, T_TRIP_IDS_SQL_CREATE, T_TRIP_IDS_SQL_INSERT, T_TRIP_IDS_SQL_DROP, getTripIdsFiles()); - if (Constants.DEBUG) MTLog.d(this, "Data: deploying DB... %s done (%s)", T_TRIP_IDS, MTLog.formatDuration(TimeUtils.currentTimeMillis() - startInMs)); + if (Constants.DEBUG) MTLog.d(this, "Data: deploying DB... '%s' done (%s)", T_TRIP_IDS, MTLog.formatDuration(TimeUtils.currentTimeMillis() - stepStartMs)); } if (notifEnabled) NotificationUtils.setProgressAndNotify(nm, nb, nId, nbTotalOperations, ++progress); + stepStartMs = TimeUtils.currentTimeMillis(); db.execSQL(T_ROUTE_DIRECTION_STOP_STATUS_SQL_CREATE); - if (Constants.DEBUG) MTLog.d(this, "Data: deploying DB... %s done (%s)", T_ROUTE_DIRECTION_STOP_STATUS, MTLog.formatDuration(TimeUtils.currentTimeMillis() - startInMs)); + if (Constants.DEBUG) MTLog.d(this, "Data: deploying DB... '%s' done (%s)", T_ROUTE_DIRECTION_STOP_STATUS, MTLog.formatDuration(TimeUtils.currentTimeMillis() - stepStartMs)); if (notifEnabled) { nb.setSmallIcon(android.R.drawable.stat_notify_sync_noanim); NotificationUtils.setProgressAndNotify(nm, nb, nId, nbTotalOperations, nbTotalOperations); diff --git a/src/main/java/org/mtransit/android/commons/provider/gtfs/GTFSScheduleTimestampsProvider.java b/src/main/java/org/mtransit/android/commons/provider/gtfs/GTFSScheduleTimestampsProvider.java index efd549c0..0fcfe29b 100644 --- a/src/main/java/org/mtransit/android/commons/provider/gtfs/GTFSScheduleTimestampsProvider.java +++ b/src/main/java/org/mtransit/android/commons/provider/gtfs/GTFSScheduleTimestampsProvider.java @@ -60,18 +60,7 @@ public static ScheduleTimestamps getScheduleTimestamps(@NonNull GTFSProvider pro final long lastDepartureInMs = TimeUnit.SECONDS.toMillis(GTFSCurrentNextProvider.getLAST_DEPARTURE_IN_SEC(context)); while (startsAt.getTimeInMillis() <= endsAtInMs) { final Calendar lookupStartAt = TimeUtils.getNewCalendar(timeZone, startsAt.getTimeInMillis()); - if (lastServiceDate != null) { - try { - while (Integer.parseInt(dateFormat.formatThreadSafe(lookupStartAt)) > lastServiceDate) { - lookupStartAt.add(Calendar.DATE, -7); // look 1 week behind - } - } catch (Exception e) { - MTLog.w(LOG_TAG, e, "Error while parsing date!"); - } - } - while (lookupStartAt.getTimeInMillis() > lastDepartureInMs) { // WHILE lookup time is after last departure DO - lookupStartAt.add(Calendar.DATE, -7); // look 1 week behind - } + GTFSStatusProvider.alignLookupStartTime(lastServiceDate, dateFormat, lookupStartAt, lastDepartureInMs); lookupDayDate = dateFormat.formatThreadSafe(lookupStartAt); lookupDayTime = timeFormat.formatThreadSafe(lookupStartAt); if (dataRequests == 0) { // IF yesterday DO override computed date & time with GTFS format for 24+ diff --git a/src/main/java/org/mtransit/android/commons/provider/gtfs/GTFSStatusProvider.java b/src/main/java/org/mtransit/android/commons/provider/gtfs/GTFSStatusProvider.java index 5bc7cdc9..4d4cf024 100644 --- a/src/main/java/org/mtransit/android/commons/provider/gtfs/GTFSStatusProvider.java +++ b/src/main/java/org/mtransit/android/commons/provider/gtfs/GTFSStatusProvider.java @@ -246,34 +246,33 @@ private static ArrayList findTimestamps(@NonNull GTFSProvide final ThreadSafeDateFormatter dateFormat = getDateFormat(context); final ThreadSafeDateFormatter timeFormat = getTimeFormat(context); final TimeZone timeZone = TimeZone.getTimeZone(AgencyUtils.getRDSAgencyTimeZone(context)); - final Calendar now = TimeUtils.getNewCalendar(timeZone, timestamp); + final Calendar startsAt = TimeUtils.getNewCalendar(timeZone, timestamp); if (lookBehindInMs > PROVIDER_PRECISION_IN_MS) { if (lookBehindInMs > 0L) { - now.add(Calendar.MILLISECOND, (int) -lookBehindInMs); + startsAt.add(Calendar.MILLISECOND, (int) -lookBehindInMs); } } else { if (PROVIDER_PRECISION_IN_MS > 0L) { - now.add(Calendar.MILLISECOND, (int) -PROVIDER_PRECISION_IN_MS); + startsAt.add(Calendar.MILLISECOND, (int) -PROVIDER_PRECISION_IN_MS); } } - now.add(Calendar.DATE, -1); // starting yesterday + startsAt.add(Calendar.DATE, -1); // starting yesterday Set dayTimestamps; String lookupDayTime; String lookupDayDate; int nbTimestamps = 0; int dataRequests = 0; + final Integer lastServiceDate = GTFSStatusProvider.findLastServiceDate(provider); final long lastDepartureInMs = TimeUnit.SECONDS.toMillis(GTFSCurrentNextProvider.getLAST_DEPARTURE_IN_SEC(context)); while (dataRequests < maxDataRequests) { - final Calendar lookupStartAt = TimeUtils.getNewCalendar(timeZone, now.getTimeInMillis()); - while (lookupStartAt.getTimeInMillis() > lastDepartureInMs) { // WHILE lookup time is after last departure DO - lookupStartAt.add(Calendar.DATE, -7); // look 1 week behind - } + final Calendar lookupStartAt = TimeUtils.getNewCalendar(timeZone, startsAt.getTimeInMillis()); + alignLookupStartTime(lastServiceDate, dateFormat, lookupStartAt, lastDepartureInMs); lookupDayDate = dateFormat.formatThreadSafe(lookupStartAt); lookupDayTime = timeFormat.formatThreadSafe(lookupStartAt); if (dataRequests == 0) { // IF yesterday DO override computed date & time with GTFS format for 24+ lookupDayTime = String.valueOf(Integer.parseInt(lookupDayTime) + TWENTY_FOUR_HOURS); } else if (dataRequests == 1) { // ELSE IF today DO - // NOTHING (keep now time) + // NOTHING (keep startsAt time) } else { // ELSE IF tomorrow or later DO lookupDayTime = MIDNIGHT; } @@ -284,10 +283,11 @@ private static ArrayList findTimestamps(@NonNull GTFSProvide rds.getStop().getId(), lookupDayDate, lookupDayTime, - now.getTimeInMillis() - lookupStartAt.getTimeInMillis() + startsAt.getTimeInMillis() - lookupStartAt.getTimeInMillis() ); - if (now.getTimeInMillis() > lookupStartAt.getTimeInMillis() // already looking at OLD schedule - && dayTimestamps.isEmpty()) { + if (dayTimestamps.isEmpty() + && startsAt.getTimeInMillis() > lookupStartAt.getTimeInMillis() // already looking at OLD schedule + ) { lookupDayDate = dateFormat.formatThreadSafe(lookupStartAt); // try 1 week before once dayTimestamps = findScheduleList( provider, @@ -296,7 +296,7 @@ private static ArrayList findTimestamps(@NonNull GTFSProvide rds.getStop().getId(), lookupDayDate, lookupDayTime, - now.getTimeInMillis() - lookupStartAt.getTimeInMillis() + startsAt.getTimeInMillis() - lookupStartAt.getTimeInMillis() ); } dataRequests++; // 1 more data request done @@ -310,10 +310,10 @@ private static ArrayList findTimestamps(@NonNull GTFSProvide } } } - if (nbTimestamps >= minUsefulResults && now.getTimeInMillis() >= minTimestampCoveredIntMs) { + if (nbTimestamps >= minUsefulResults && startsAt.getTimeInMillis() >= minTimestampCoveredIntMs) { break; } - now.add(Calendar.DATE, +1); // NEXT DAY + startsAt.add(Calendar.DATE, +1); // NEXT DAY } if (FeatureFlags.F_EXPORT_STRINGS || FeatureFlags.F_EXPORT_SCHEDULE_STRINGS) { allTimestamps = GTFSStringsUtils.updateStrings(allTimestamps, provider); @@ -324,6 +324,21 @@ private static ArrayList findTimestamps(@NonNull GTFSProvide return allTimestamps; } + protected static void alignLookupStartTime(Integer lastServiceDate, ThreadSafeDateFormatter dateFormat, Calendar lookupStartAt, long lastDepartureInMs) { + if (lastServiceDate != null) { + try { + while (Integer.parseInt(dateFormat.formatThreadSafe(lookupStartAt)) > lastServiceDate) { + lookupStartAt.add(Calendar.DATE, -7); // look 1 week behind + } + } catch (Exception e) { + MTLog.w(LOG_TAG, e, "Error while parsing date!"); + } + } + while (lookupStartAt.getTimeInMillis() > lastDepartureInMs) { // WHILE lookup time is after last departure DO + lookupStartAt.add(Calendar.DATE, -7); // look 1 week behind + } + } + @Nullable private static String stopScheduleRawFileFormat = null; @@ -584,24 +599,23 @@ private static ArrayList findFrequencies(@NonNull GTFSProvid final ThreadSafeDateFormatter dateFormat = getDateFormat(context); final ThreadSafeDateFormatter timeFormat = getTimeFormat(context); final TimeZone timeZone = TimeZone.getTimeZone(AgencyUtils.getRDSAgencyTimeZone(context)); - final Calendar now = TimeUtils.getNewCalendar(timeZone, timestamp); - now.add(Calendar.DATE, -1); // starting yesterday + final Calendar startsAt = TimeUtils.getNewCalendar(timeZone, timestamp); + startsAt.add(Calendar.DATE, -1); // starting yesterday HashSet dayFrequencies; String lookupDayTime; String lookupDayDate; int dataRequests = 0; + final Integer lastServiceDate = GTFSStatusProvider.findLastServiceDate(provider); final long lastDepartureInMs = TimeUnit.SECONDS.toMillis(GTFSCurrentNextProvider.getLAST_DEPARTURE_IN_SEC(context)); while (dataRequests < maxDataRequests) { - final Calendar lookupTime = TimeUtils.getNewCalendar(timeZone, now.getTimeInMillis()); - while (lookupTime.getTimeInMillis() > lastDepartureInMs) { // WHILE lookup time is after last departure DO - lookupTime.add(Calendar.DATE, -7); // look 1 week behind - } - lookupDayDate = dateFormat.formatThreadSafe(lookupTime); - lookupDayTime = timeFormat.formatThreadSafe(lookupTime); + final Calendar lookupStartAt = TimeUtils.getNewCalendar(timeZone, startsAt.getTimeInMillis()); + alignLookupStartTime(lastServiceDate, dateFormat, lookupStartAt, lastDepartureInMs); + lookupDayDate = dateFormat.formatThreadSafe(lookupStartAt); + lookupDayTime = timeFormat.formatThreadSafe(lookupStartAt); if (dataRequests == 0) { // IF yesterday DO override computed date & time with GTFS format for 24+ lookupDayTime = String.valueOf(Integer.parseInt(lookupDayTime) + TWENTY_FOUR_HOURS); } else if (dataRequests == 1) { // ELSE IF today DO - // NOTHING (keep now time) + // NOTHING (keep startsAt time) } else { // ELSE IF tomorrow or later DO lookupDayTime = MIDNIGHT; } @@ -611,21 +625,21 @@ private static ArrayList findFrequencies(@NonNull GTFSProvid rds.getDirection().getId(), lookupDayDate, lookupDayTime, - now.getTimeInMillis() - lookupTime.getTimeInMillis() + startsAt.getTimeInMillis() - lookupStartAt.getTimeInMillis() ); if (dayFrequencies.isEmpty() - && now.getTimeInMillis() > lookupTime.getTimeInMillis() // already looking at OLD schedule + && startsAt.getTimeInMillis() > lookupStartAt.getTimeInMillis() // already looking at OLD schedule && MIDNIGHT.equals(lookupDayTime) // not a partial schedule ) { - lookupTime.add(Calendar.DATE, -7); // look 1 week behind - lookupDayDate = dateFormat.formatThreadSafe(lookupTime); // try 1 week before once + lookupStartAt.add(Calendar.DATE, -7); // look 1 week behind + lookupDayDate = dateFormat.formatThreadSafe(lookupStartAt); // try 1 week before once dayFrequencies = findFrequencyList( provider, rds.getRoute().getId(), rds.getDirection().getId(), lookupDayDate, lookupDayTime, - now.getTimeInMillis() - lookupTime.getTimeInMillis() + startsAt.getTimeInMillis() - lookupStartAt.getTimeInMillis() ); } dataRequests++; // 1 more data request done @@ -634,10 +648,10 @@ private static ArrayList findFrequencies(@NonNull GTFSProvid allFrequencies.add(dayFrequency); } } - if (now.getTimeInMillis() >= minTimestampCovered) { + if (startsAt.getTimeInMillis() >= minTimestampCovered) { break; } - now.add(Calendar.DATE, +1); // NEXT DAY + startsAt.add(Calendar.DATE, +1); // NEXT DAY } return allFrequencies; }