Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Original file line number Diff line number Diff line change
Expand Up @@ -5,6 +5,7 @@ import android.database.sqlite.SQLiteDatabase
import androidx.core.database.sqlite.transaction
import org.mtransit.android.commons.FileUtils
import org.mtransit.android.commons.MTLog
import org.mtransit.android.commons.provider.gtfs.GTFSProviderDbHelper.DB_NAME
import org.mtransit.commons.FeatureFlags
import org.mtransit.commons.GTFSCommons
import org.mtransit.commons.sql.SQLUtils
Expand Down Expand Up @@ -67,7 +68,7 @@ object GTFSProviderDBHelperUtils : MTLog.Loggable {
transaction {
execSQL(sqlDrop) // drop if exists
execSQL(sqlCreate) // create if not exists
for (file in files) {
files.forEach { file ->
try {
openRawResource(file).use { inputStream ->
InputStreamReader(inputStream, FileUtils.getUTF8()).use { inputStreamReader ->
Expand All @@ -87,26 +88,22 @@ object GTFSProviderDBHelperUtils : MTLog.Loggable {
try {
execSQL(sql)
} catch (e: Exception) {
MTLog.w(
this,
e,
"ERROR while executing '$sql' on database '${GTFSProviderDbHelper.DB_NAME}' table '$table' file '$file'!"
)
MTLog.w(this, e, "ERROR while executing '$sql' on database '$DB_NAME' table '$table' file '$file'!")
throw e
}
}
}
}
}
} catch (e: Exception) {
MTLog.w(this, e, "ERROR while copying the database '${GTFSProviderDbHelper.DB_NAME}' table '$table' file '$file'!")
MTLog.w(this, e, "ERROR while copying the database '$DB_NAME' table '$table' file '$file'!")
return false
}
}
}
return true
} catch (e: Exception) {
MTLog.w(this, e, "ERROR while copying the database '${GTFSProviderDbHelper.DB_NAME}' table '$table' file!")
MTLog.w(this, e, "ERROR while copying the database '$DB_NAME' table '$table' file!")
return false
}
}
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -189,6 +189,7 @@ private void initAllDbTables(@NonNull SQLiteDatabase db, boolean upgrade) {
MTLog.i(this, "Data: deploying DB...");
final int nId = TimeUtils.currentTimeSec();
final long startInMs = TimeUtils.currentTimeMillis();
long stepStartMs;
int nbTotalOperations = 7;
if (FeatureFlags.F_EXPORT_TRIP_ID) nbTotalOperations++;
if (FeatureFlags.F_EXPORT_SERVICE_ID_INTS) nbTotalOperations++;
Expand All @@ -212,47 +213,57 @@ private void initAllDbTables(@NonNull SQLiteDatabase db, boolean upgrade) {
if (notifEnabled) NotificationUtils.setProgressAndNotify(nm, nb, nId, nbTotalOperations, ++progress);
final Map<Integer, String> allStrings = new HashMap<>();
if (FeatureFlags.F_EXPORT_STRINGS || FeatureFlags.F_EXPORT_SCHEDULE_STRINGS) {
stepStartMs = TimeUtils.currentTimeMillis();
initDbTableWithRetry(context, db, T_STRINGS, T_STRINGS_SQL_CREATE, T_STRINGS_SQL_INSERT, T_STRINGS_SQL_DROP, getStringsFiles(), 0, 0, null, null,
(id, string) -> {
allStrings.put(id, string);
return kotlin.Unit.INSTANCE;
}
); // 1st
if (Constants.DEBUG) MTLog.d(this, "Data: deploying DB... %s done (%s)", T_STRINGS, MTLog.formatDuration(TimeUtils.currentTimeMillis() - startInMs));
if (Constants.DEBUG) MTLog.d(this, "Data: deploying DB... '%s' done (%s)", T_STRINGS, MTLog.formatDuration(TimeUtils.currentTimeMillis() - stepStartMs));
}
if (notifEnabled) NotificationUtils.setProgressAndNotify(nm, nb, nId, nbTotalOperations, ++progress);
stepStartMs = TimeUtils.currentTimeMillis();
initDbTableWithRetry(context, db, T_ROUTE, T_ROUTE_SQL_CREATE, T_ROUTE_SQL_INSERT, T_ROUTE_SQL_DROP, getRouteFiles(), 0, 0, allStrings, T_ROUTE_STRINGS_COLUMN_IDX);
if (Constants.DEBUG) MTLog.d(this, "Data: deploying DB... %s done (%s)", T_ROUTE, MTLog.formatDuration(TimeUtils.currentTimeMillis() - startInMs));
if (Constants.DEBUG) MTLog.d(this, "Data: deploying DB... '%s' done (%s)", T_ROUTE, MTLog.formatDuration(TimeUtils.currentTimeMillis() - stepStartMs));
if (notifEnabled) NotificationUtils.setProgressAndNotify(nm, nb, nId, nbTotalOperations, ++progress);
stepStartMs = TimeUtils.currentTimeMillis();
initDbTableWithRetry(context, db, T_DIRECTION, T_DIRECTION_SQL_CREATE, T_DIRECTION_SQL_INSERT, T_DIRECTION_SQL_DROP, getDirectionFiles(), 0, 0, allStrings, T_DIRECTION_STRINGS_COLUMN_IDX);
if (Constants.DEBUG) MTLog.d(this, "Data: deploying DB... %s done (%s)", T_DIRECTION, MTLog.formatDuration(TimeUtils.currentTimeMillis() - startInMs));
if (Constants.DEBUG) MTLog.d(this, "Data: deploying DB... '%s' done (%s)", T_DIRECTION, MTLog.formatDuration(TimeUtils.currentTimeMillis() - stepStartMs));
if (notifEnabled) NotificationUtils.setProgressAndNotify(nm, nb, nId, nbTotalOperations, ++progress);
stepStartMs = TimeUtils.currentTimeMillis();
initDbTableWithRetry(context, db, T_STOP, T_STOP_SQL_CREATE, T_STOP_SQL_INSERT, T_STOP_SQL_DROP, getStopFiles(), 0, 0, allStrings, T_STOP_STRINGS_COLUMN_IDX);
if (Constants.DEBUG) MTLog.d(this, "Data: deploying DB... %s done (%s)", T_STOP, MTLog.formatDuration(TimeUtils.currentTimeMillis() - startInMs));
if (Constants.DEBUG) MTLog.d(this, "Data: deploying DB... '%s' done (%s)", T_STOP, MTLog.formatDuration(TimeUtils.currentTimeMillis() - stepStartMs));
if (notifEnabled) NotificationUtils.setProgressAndNotify(nm, nb, nId, nbTotalOperations, ++progress);
stepStartMs = TimeUtils.currentTimeMillis();
initDbTableWithRetry(context, db, T_DIRECTION_STOPS, T_DIRECTION_STOPS_SQL_CREATE, T_DIRECTION_STOPS_SQL_INSERT, T_DIRECTION_STOPS_SQL_DROP, getDirectionStopsFiles());
if (Constants.DEBUG) MTLog.d(this, "Data: deploying DB... %s done (%s)", T_DIRECTION_STOPS, MTLog.formatDuration(TimeUtils.currentTimeMillis() - startInMs));
if (Constants.DEBUG) MTLog.d(this, "Data: deploying DB... '%s' done (%s)", T_DIRECTION_STOPS, MTLog.formatDuration(TimeUtils.currentTimeMillis() - stepStartMs));
if (FeatureFlags.F_EXPORT_TRIP_ID) {
if (notifEnabled) NotificationUtils.setProgressAndNotify(nm, nb, nId, nbTotalOperations, ++progress);
stepStartMs = TimeUtils.currentTimeMillis();
initDbTableWithRetry(context, db, T_TRIP, T_TRIP_SQL_CREATE, T_TRIP_SQL_INSERT, T_TRIP_SQL_DROP, getTripFiles(), T_TRIP_SAME_COLUMNS_COUNT, T_TRIP_OTHER_COLUMNS_COUNT);
if (Constants.DEBUG) MTLog.d(this, "Data: deploying DB... %s done (%s)", T_TRIP, MTLog.formatDuration(TimeUtils.currentTimeMillis() - startInMs));
if (Constants.DEBUG) MTLog.d(this, "Data: deploying DB... '%s' done (%s)", T_TRIP, MTLog.formatDuration(TimeUtils.currentTimeMillis() - stepStartMs));
}
if (FeatureFlags.F_EXPORT_SERVICE_ID_INTS) {
if (notifEnabled) NotificationUtils.setProgressAndNotify(nm, nb, nId, nbTotalOperations, ++progress);
stepStartMs = TimeUtils.currentTimeMillis();
initDbTableWithRetry(context, db, T_SERVICE_IDS, T_SERVICE_IDS_SQL_CREATE, T_SERVICE_IDS_SQL_INSERT, T_SERVICE_IDS_SQL_DROP, getServiceIdsFiles());
if (Constants.DEBUG) MTLog.d(this, "Data: deploying DB... %s done (%s)", T_SERVICE_IDS, MTLog.formatDuration(TimeUtils.currentTimeMillis() - startInMs));
if (Constants.DEBUG) MTLog.d(this, "Data: deploying DB... '%s' done (%s)", T_SERVICE_IDS, MTLog.formatDuration(TimeUtils.currentTimeMillis() - stepStartMs));
}
if (notifEnabled) NotificationUtils.setProgressAndNotify(nm, nb, nId, nbTotalOperations, ++progress);
stepStartMs = TimeUtils.currentTimeMillis();
initDbTableWithRetry(context, db, T_SERVICE_DATES, T_SERVICE_DATES_SQL_CREATE, T_SERVICE_DATES_SQL_INSERT, T_SERVICE_DATES_SQL_DROP, getServiceDatesFiles(), T_SERVICE_DATES_SAME_COLUMNS_COUNT, T_SERVICE_DATES_OTHER_COLUMNS_COUNT);
if (Constants.DEBUG) MTLog.d(this, "Data: deploying DB... %s done (%s)", T_SERVICE_DATES, MTLog.formatDuration(TimeUtils.currentTimeMillis() - startInMs));
if (Constants.DEBUG) MTLog.d(this, "Data: deploying DB... '%s' done (%s)", T_SERVICE_DATES, MTLog.formatDuration(TimeUtils.currentTimeMillis() - stepStartMs));
if (FeatureFlags.F_EXPORT_TRIP_ID_INTS) {
if (notifEnabled) NotificationUtils.setProgressAndNotify(nm, nb, nId, nbTotalOperations, ++progress);
stepStartMs = TimeUtils.currentTimeMillis();
initDbTableWithRetry(context, db, T_TRIP_IDS, T_TRIP_IDS_SQL_CREATE, T_TRIP_IDS_SQL_INSERT, T_TRIP_IDS_SQL_DROP, getTripIdsFiles());
if (Constants.DEBUG) MTLog.d(this, "Data: deploying DB... %s done (%s)", T_TRIP_IDS, MTLog.formatDuration(TimeUtils.currentTimeMillis() - startInMs));
if (Constants.DEBUG) MTLog.d(this, "Data: deploying DB... '%s' done (%s)", T_TRIP_IDS, MTLog.formatDuration(TimeUtils.currentTimeMillis() - stepStartMs));
}
if (notifEnabled) NotificationUtils.setProgressAndNotify(nm, nb, nId, nbTotalOperations, ++progress);
stepStartMs = TimeUtils.currentTimeMillis();
db.execSQL(T_ROUTE_DIRECTION_STOP_STATUS_SQL_CREATE);
if (Constants.DEBUG) MTLog.d(this, "Data: deploying DB... %s done (%s)", T_ROUTE_DIRECTION_STOP_STATUS, MTLog.formatDuration(TimeUtils.currentTimeMillis() - startInMs));
if (Constants.DEBUG) MTLog.d(this, "Data: deploying DB... '%s' done (%s)", T_ROUTE_DIRECTION_STOP_STATUS, MTLog.formatDuration(TimeUtils.currentTimeMillis() - stepStartMs));
if (notifEnabled) {
nb.setSmallIcon(android.R.drawable.stat_notify_sync_noanim);
NotificationUtils.setProgressAndNotify(nm, nb, nId, nbTotalOperations, nbTotalOperations);
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -60,18 +60,7 @@ public static ScheduleTimestamps getScheduleTimestamps(@NonNull GTFSProvider pro
final long lastDepartureInMs = TimeUnit.SECONDS.toMillis(GTFSCurrentNextProvider.getLAST_DEPARTURE_IN_SEC(context));
while (startsAt.getTimeInMillis() <= endsAtInMs) {
final Calendar lookupStartAt = TimeUtils.getNewCalendar(timeZone, startsAt.getTimeInMillis());
if (lastServiceDate != null) {
try {
while (Integer.parseInt(dateFormat.formatThreadSafe(lookupStartAt)) > lastServiceDate) {
lookupStartAt.add(Calendar.DATE, -7); // look 1 week behind
}
} catch (Exception e) {
MTLog.w(LOG_TAG, e, "Error while parsing date!");
}
}
while (lookupStartAt.getTimeInMillis() > lastDepartureInMs) { // WHILE lookup time is after last departure DO
lookupStartAt.add(Calendar.DATE, -7); // look 1 week behind
}
GTFSStatusProvider.alignLookupStartTime(lastServiceDate, dateFormat, lookupStartAt, lastDepartureInMs);
lookupDayDate = dateFormat.formatThreadSafe(lookupStartAt);
lookupDayTime = timeFormat.formatThreadSafe(lookupStartAt);
if (dataRequests == 0) { // IF yesterday DO override computed date & time with GTFS format for 24+
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -246,34 +246,33 @@ private static ArrayList<Schedule.Timestamp> findTimestamps(@NonNull GTFSProvide
final ThreadSafeDateFormatter dateFormat = getDateFormat(context);
final ThreadSafeDateFormatter timeFormat = getTimeFormat(context);
final TimeZone timeZone = TimeZone.getTimeZone(AgencyUtils.getRDSAgencyTimeZone(context));
final Calendar now = TimeUtils.getNewCalendar(timeZone, timestamp);
final Calendar startsAt = TimeUtils.getNewCalendar(timeZone, timestamp);
if (lookBehindInMs > PROVIDER_PRECISION_IN_MS) {
if (lookBehindInMs > 0L) {
now.add(Calendar.MILLISECOND, (int) -lookBehindInMs);
startsAt.add(Calendar.MILLISECOND, (int) -lookBehindInMs);
}
} else {
if (PROVIDER_PRECISION_IN_MS > 0L) {
now.add(Calendar.MILLISECOND, (int) -PROVIDER_PRECISION_IN_MS);
startsAt.add(Calendar.MILLISECOND, (int) -PROVIDER_PRECISION_IN_MS);
}
}
now.add(Calendar.DATE, -1); // starting yesterday
startsAt.add(Calendar.DATE, -1); // starting yesterday
Set<Schedule.Timestamp> dayTimestamps;
String lookupDayTime;
String lookupDayDate;
int nbTimestamps = 0;
int dataRequests = 0;
final Integer lastServiceDate = GTFSStatusProvider.findLastServiceDate(provider);
final long lastDepartureInMs = TimeUnit.SECONDS.toMillis(GTFSCurrentNextProvider.getLAST_DEPARTURE_IN_SEC(context));
while (dataRequests < maxDataRequests) {
final Calendar lookupStartAt = TimeUtils.getNewCalendar(timeZone, now.getTimeInMillis());
while (lookupStartAt.getTimeInMillis() > lastDepartureInMs) { // WHILE lookup time is after last departure DO
lookupStartAt.add(Calendar.DATE, -7); // look 1 week behind
}
final Calendar lookupStartAt = TimeUtils.getNewCalendar(timeZone, startsAt.getTimeInMillis());
alignLookupStartTime(lastServiceDate, dateFormat, lookupStartAt, lastDepartureInMs);
lookupDayDate = dateFormat.formatThreadSafe(lookupStartAt);
lookupDayTime = timeFormat.formatThreadSafe(lookupStartAt);
if (dataRequests == 0) { // IF yesterday DO override computed date & time with GTFS format for 24+
lookupDayTime = String.valueOf(Integer.parseInt(lookupDayTime) + TWENTY_FOUR_HOURS);
} else if (dataRequests == 1) { // ELSE IF today DO
// NOTHING (keep now time)
// NOTHING (keep startsAt time)
} else { // ELSE IF tomorrow or later DO
lookupDayTime = MIDNIGHT;
}
Expand All @@ -284,10 +283,11 @@ private static ArrayList<Schedule.Timestamp> findTimestamps(@NonNull GTFSProvide
rds.getStop().getId(),
lookupDayDate,
lookupDayTime,
now.getTimeInMillis() - lookupStartAt.getTimeInMillis()
startsAt.getTimeInMillis() - lookupStartAt.getTimeInMillis()
);
if (now.getTimeInMillis() > lookupStartAt.getTimeInMillis() // already looking at OLD schedule
&& dayTimestamps.isEmpty()) {
if (dayTimestamps.isEmpty()
&& startsAt.getTimeInMillis() > lookupStartAt.getTimeInMillis() // already looking at OLD schedule
) {
lookupDayDate = dateFormat.formatThreadSafe(lookupStartAt); // try 1 week before once
dayTimestamps = findScheduleList(
provider,
Expand All @@ -296,7 +296,7 @@ private static ArrayList<Schedule.Timestamp> findTimestamps(@NonNull GTFSProvide
rds.getStop().getId(),
lookupDayDate,
lookupDayTime,
now.getTimeInMillis() - lookupStartAt.getTimeInMillis()
startsAt.getTimeInMillis() - lookupStartAt.getTimeInMillis()
);
}
dataRequests++; // 1 more data request done
Expand All @@ -310,10 +310,10 @@ private static ArrayList<Schedule.Timestamp> findTimestamps(@NonNull GTFSProvide
}
}
}
if (nbTimestamps >= minUsefulResults && now.getTimeInMillis() >= minTimestampCoveredIntMs) {
if (nbTimestamps >= minUsefulResults && startsAt.getTimeInMillis() >= minTimestampCoveredIntMs) {
break;
}
now.add(Calendar.DATE, +1); // NEXT DAY
startsAt.add(Calendar.DATE, +1); // NEXT DAY
}
if (FeatureFlags.F_EXPORT_STRINGS || FeatureFlags.F_EXPORT_SCHEDULE_STRINGS) {
allTimestamps = GTFSStringsUtils.updateStrings(allTimestamps, provider);
Expand All @@ -324,6 +324,21 @@ private static ArrayList<Schedule.Timestamp> findTimestamps(@NonNull GTFSProvide
return allTimestamps;
}

protected static void alignLookupStartTime(Integer lastServiceDate, ThreadSafeDateFormatter dateFormat, Calendar lookupStartAt, long lastDepartureInMs) {
if (lastServiceDate != null) {
try {
while (Integer.parseInt(dateFormat.formatThreadSafe(lookupStartAt)) > lastServiceDate) {
lookupStartAt.add(Calendar.DATE, -7); // look 1 week behind
}
} catch (Exception e) {
MTLog.w(LOG_TAG, e, "Error while parsing date!");
}
}
while (lookupStartAt.getTimeInMillis() > lastDepartureInMs) { // WHILE lookup time is after last departure DO
lookupStartAt.add(Calendar.DATE, -7); // look 1 week behind
}
}

@Nullable
private static String stopScheduleRawFileFormat = null;

Expand Down Expand Up @@ -584,24 +599,23 @@ private static ArrayList<Schedule.Frequency> findFrequencies(@NonNull GTFSProvid
final ThreadSafeDateFormatter dateFormat = getDateFormat(context);
final ThreadSafeDateFormatter timeFormat = getTimeFormat(context);
final TimeZone timeZone = TimeZone.getTimeZone(AgencyUtils.getRDSAgencyTimeZone(context));
final Calendar now = TimeUtils.getNewCalendar(timeZone, timestamp);
now.add(Calendar.DATE, -1); // starting yesterday
final Calendar startsAt = TimeUtils.getNewCalendar(timeZone, timestamp);
startsAt.add(Calendar.DATE, -1); // starting yesterday
HashSet<Schedule.Frequency> dayFrequencies;
String lookupDayTime;
String lookupDayDate;
int dataRequests = 0;
final Integer lastServiceDate = GTFSStatusProvider.findLastServiceDate(provider);
final long lastDepartureInMs = TimeUnit.SECONDS.toMillis(GTFSCurrentNextProvider.getLAST_DEPARTURE_IN_SEC(context));
while (dataRequests < maxDataRequests) {
final Calendar lookupTime = TimeUtils.getNewCalendar(timeZone, now.getTimeInMillis());
while (lookupTime.getTimeInMillis() > lastDepartureInMs) { // WHILE lookup time is after last departure DO
lookupTime.add(Calendar.DATE, -7); // look 1 week behind
}
lookupDayDate = dateFormat.formatThreadSafe(lookupTime);
lookupDayTime = timeFormat.formatThreadSafe(lookupTime);
final Calendar lookupStartAt = TimeUtils.getNewCalendar(timeZone, startsAt.getTimeInMillis());
alignLookupStartTime(lastServiceDate, dateFormat, lookupStartAt, lastDepartureInMs);
lookupDayDate = dateFormat.formatThreadSafe(lookupStartAt);
lookupDayTime = timeFormat.formatThreadSafe(lookupStartAt);
if (dataRequests == 0) { // IF yesterday DO override computed date & time with GTFS format for 24+
lookupDayTime = String.valueOf(Integer.parseInt(lookupDayTime) + TWENTY_FOUR_HOURS);
} else if (dataRequests == 1) { // ELSE IF today DO
// NOTHING (keep now time)
// NOTHING (keep startsAt time)
} else { // ELSE IF tomorrow or later DO
lookupDayTime = MIDNIGHT;
}
Expand All @@ -611,21 +625,21 @@ private static ArrayList<Schedule.Frequency> findFrequencies(@NonNull GTFSProvid
rds.getDirection().getId(),
lookupDayDate,
lookupDayTime,
now.getTimeInMillis() - lookupTime.getTimeInMillis()
startsAt.getTimeInMillis() - lookupStartAt.getTimeInMillis()
);
if (dayFrequencies.isEmpty()
&& now.getTimeInMillis() > lookupTime.getTimeInMillis() // already looking at OLD schedule
&& startsAt.getTimeInMillis() > lookupStartAt.getTimeInMillis() // already looking at OLD schedule
&& MIDNIGHT.equals(lookupDayTime) // not a partial schedule
) {
lookupTime.add(Calendar.DATE, -7); // look 1 week behind
lookupDayDate = dateFormat.formatThreadSafe(lookupTime); // try 1 week before once
lookupStartAt.add(Calendar.DATE, -7); // look 1 week behind
lookupDayDate = dateFormat.formatThreadSafe(lookupStartAt); // try 1 week before once
dayFrequencies = findFrequencyList(
provider,
rds.getRoute().getId(),
rds.getDirection().getId(),
lookupDayDate,
lookupDayTime,
now.getTimeInMillis() - lookupTime.getTimeInMillis()
startsAt.getTimeInMillis() - lookupStartAt.getTimeInMillis()
);
}
dataRequests++; // 1 more data request done
Expand All @@ -634,10 +648,10 @@ private static ArrayList<Schedule.Frequency> findFrequencies(@NonNull GTFSProvid
allFrequencies.add(dayFrequency);
}
}
if (now.getTimeInMillis() >= minTimestampCovered) {
if (startsAt.getTimeInMillis() >= minTimestampCovered) {
break;
}
now.add(Calendar.DATE, +1); // NEXT DAY
startsAt.add(Calendar.DATE, +1); // NEXT DAY
}
return allFrequencies;
}
Expand Down