20 video conversion for web view (immich-app#200)

* Added job for video conversion every 1 minute

* Handle get video as mp4 on the web

* Auto play video on web on hovered

* Added video player

* Added animation and video duration to thumbnail player

* Fixed issue with video not playing on hover

* Added animation when loading thumbnail

Author: alextran1502

server/src/api-v1/asset/asset.module.ts

@@ -38,4 +38,4 @@ import { CommunicationModule } from '../communication/communication.module';
   providers: [AssetService, AssetOptimizeService, BackgroundTaskService],
   exports: [],
 })
-export class AssetModule {}
+export class AssetModule { }

server/src/api-v1/asset/asset.service.ts

@@ -11,6 +11,7 @@ import { Response as Res } from 'express';
 import { promisify } from 'util';
 import { DeleteAssetDto } from './dto/delete-asset.dto';
 import { SearchAssetDto } from './dto/search-asset.dto';
+import ffmpeg from 'fluent-ffmpeg';
 
 const fileInfo = promisify(stat);
 
@@ -185,7 +186,15 @@ export class AssetService {
 
     } else if (asset.type == AssetType.VIDEO) {
       // Handle Video
-      const { size } = await fileInfo(asset.originalPath);
+      let videoPath = asset.originalPath;
+      let mimeType = asset.mimeType;
+
+      if (query.isWeb && asset.mimeType == 'video/quicktime') {
+        videoPath = asset.encodedVideoPath == '' ? asset.originalPath : asset.encodedVideoPath;
+        mimeType = asset.encodedVideoPath == '' ? asset.mimeType : 'video/mp4';
+      }
+
+      const { size } = await fileInfo(videoPath);
       const range = headers.range;
 
       if (range) {
@@ -220,20 +229,22 @@ export class AssetService {
           'Content-Range': `bytes ${start}-${end}/${size}`,
           'Accept-Ranges': 'bytes',
           'Content-Length': end - start + 1,
-          'Content-Type': asset.mimeType,
+          'Content-Type': mimeType,
         });
 
-        const videoStream = createReadStream(asset.originalPath, { start: start, end: end });
+
+        const videoStream = createReadStream(videoPath, { start: start, end: end });
 
         return new StreamableFile(videoStream);
 
 
       } else {
+
         res.set({
-          'Content-Type': asset.mimeType,
+          'Content-Type': mimeType,
         });
 
-        return new StreamableFile(createReadStream(asset.originalPath));
+        return new StreamableFile(createReadStream(videoPath));
       }
     }
   }

server/src/api-v1/asset/entities/asset.entity.ts

@@ -29,6 +29,9 @@ export class AssetEntity {
   @Column({ nullable: true })
   webpPath: string;
 
+  @Column({ nullable: true })
+  encodedVideoPath: string;
+
   @Column()
   createdAt: string;
 

server/src/app.module.ts

@@ -65,7 +65,7 @@ import { ScheduleTasksModule } from './modules/schedule-tasks/schedule-tasks.mod
 export class AppModule implements NestModule {
   configure(consumer: MiddlewareConsumer): void {
     if (process.env.NODE_ENV == 'development') {
-      consumer.apply(AppLoggerMiddleware).forRoutes('*');
+      // consumer.apply(AppLoggerMiddleware).forRoutes('*');
     }
   }
 }

server/src/migration/1654299904583-UpdateAssetTableWithEncodeVideoPath.ts

@@ -0,0 +1,17 @@
+import { MigrationInterface, QueryRunner } from "typeorm";
+
+export class UpdateAssetTableWithEncodeVideoPath1654299904583 implements MigrationInterface {
+  public async up(queryRunner: QueryRunner): Promise<void> {
+    await queryRunner.query(`
+      alter table assets
+        add column if not exists "encodedVideoPath" varchar default '';
+      `)
+  }
+
+  public async down(queryRunner: QueryRunner): Promise<void> {
+    await queryRunner.query(`
+      alter table assets
+        drop column if exists "encodedVideoPath";     
+      `);
+  }
+}

server/src/modules/background-task/background-task.module.ts

@@ -17,9 +17,10 @@ import { BackgroundTaskService } from './background-task.service';
         removeOnFail: false,
       },
     }),
+
     TypeOrmModule.forFeature([AssetEntity, ExifEntity, SmartInfoEntity]),
   ],
   providers: [BackgroundTaskService, BackgroundTaskProcessor],
   exports: [BackgroundTaskService],
 })
-export class BackgroundTaskModule {}
+export class BackgroundTaskModule { }

server/src/modules/image-optimize/image-optimize.module.ts

@@ -33,4 +33,4 @@ import { AssetOptimizeService } from './image-optimize.service';
   providers: [AssetOptimizeService, ImageOptimizeProcessor, BackgroundTaskService],
   exports: [AssetOptimizeService],
 })
-export class ImageOptimizeModule {}
+export class ImageOptimizeModule { }

server/src/modules/schedule-tasks/schedule-tasks.module.ts

@@ -1,12 +1,30 @@
+import { BullModule } from '@nestjs/bull';
 import { Module } from '@nestjs/common';
 import { TypeOrmModule } from '@nestjs/typeorm';
+import { AssetModule } from '../../api-v1/asset/asset.module';
 import { AssetEntity } from '../../api-v1/asset/entities/asset.entity';
 import { ImageConversionService } from './image-conversion.service';
+import { VideoConversionProcessor } from './video-conversion.processor';
+import { VideoConversionService } from './video-conversion.service';
 
 @Module({
   imports: [
     TypeOrmModule.forFeature([AssetEntity]),
+
+    BullModule.registerQueue({
+      settings: {},
+      name: 'video-conversion',
+      limiter: {
+        max: 1,
+        duration: 60000
+      },
+      defaultJobOptions: {
+        attempts: 3,
+        removeOnComplete: true,
+        removeOnFail: false,
+      },
+    }),
   ],
-  providers: [ImageConversionService],
+  providers: [ImageConversionService, VideoConversionService, VideoConversionProcessor,],
 })
 export class ScheduleTasksModule { }

server/src/modules/schedule-tasks/video-conversion.processor.ts

@@ -0,0 +1,56 @@
+import { Process, Processor } from '@nestjs/bull';
+import { InjectRepository } from '@nestjs/typeorm';
+import { Job } from 'bull';
+import { Repository } from 'typeorm';
+import { AssetEntity } from '../../api-v1/asset/entities/asset.entity';
+import { existsSync, mkdirSync } from 'fs';
+import { APP_UPLOAD_LOCATION } from '../../constants/upload_location.constant';
+import ffmpeg from 'fluent-ffmpeg';
+import { Logger } from '@nestjs/common';
+
+@Processor('video-conversion')
+export class VideoConversionProcessor {
+
+  constructor(
+    @InjectRepository(AssetEntity)
+    private assetRepository: Repository<AssetEntity>,
+  ) { }
+
+  @Process('to-mp4')
+  async convertToMp4(job: Job) {
+    const { asset }: { asset: AssetEntity } = job.data;
+
+    const basePath = APP_UPLOAD_LOCATION;
+    const encodedVideoPath = `${basePath}/${asset.userId}/encoded-video`;
+
+    if (!existsSync(encodedVideoPath)) {
+      mkdirSync(encodedVideoPath, { recursive: true });
+    }
+
+    const latestAssetInfo = await this.assetRepository.findOne({ id: asset.id });
+    const savedEncodedPath = encodedVideoPath + "/" + latestAssetInfo.id + '.mp4'
+
+    if (latestAssetInfo.encodedVideoPath == '') {
+      ffmpeg(latestAssetInfo.originalPath)
+        .outputOptions([
+          '-crf 23',
+          '-preset ultrafast',
+          '-vcodec libx264',
+          '-acodec mp3',
+          '-vf scale=1280:-2'
+        ])
+        .output(savedEncodedPath)
+        .on('start', () => Logger.log("Start Converting", 'VideoConversionMOV2MP4'))
+        .on('error', (a, b, c) => {
+          Logger.error('Cannot Convert Video', 'VideoConversionMOV2MP4')
+          console.log(a, b, c)
+        })
+        .on('end', async () => {
+          Logger.log(`Converting Success ${latestAssetInfo.id}`, 'VideoConversionMOV2MP4')
+          await this.assetRepository.update({ id: latestAssetInfo.id }, { encodedVideoPath: savedEncodedPath });
+        }).run();
+    }
+
+    return {}
+  }
+}

server/src/modules/schedule-tasks/video-conversion.service.ts

@@ -0,0 +1,50 @@
+import { Injectable, Logger } from '@nestjs/common';
+import { Cron, CronExpression } from '@nestjs/schedule';
+import { InjectRepository } from '@nestjs/typeorm';
+import { Repository } from 'typeorm';
+import { AssetEntity } from '../../api-v1/asset/entities/asset.entity';
+import sharp from 'sharp';
+import ffmpeg from 'fluent-ffmpeg';
+import { APP_UPLOAD_LOCATION } from '../../constants/upload_location.constant';
+import { existsSync, mkdirSync } from 'fs';
+import { InjectQueue } from '@nestjs/bull/dist/decorators';
+import { Queue } from 'bull';
+import { randomUUID } from 'crypto';
+
+@Injectable()
+export class VideoConversionService {
+
+
+  constructor(
+    @InjectRepository(AssetEntity)
+    private assetRepository: Repository<AssetEntity>,
+
+    @InjectQueue('video-conversion')
+    private videoEncodingQueue: Queue
+  ) { }
+
+
+  // time ffmpeg -i 15065f4a-47ff-4aed-8c3e-c9fcf1840531.mov -crf 35 -preset ultrafast -vcodec libx264 -acodec mp3 -vf "scale=1280:-1" 15065f4a-47ff-4aed-8c3e-c9fcf1840531.mp4
+  @Cron(CronExpression.EVERY_MINUTE
+    , {
+      name: 'video-encoding'
+    })
+  async mp4Conversion() {
+    const assets = await this.assetRepository.find({
+      where: {
+        type: 'VIDEO',
+        mimeType: 'video/quicktime',
+        encodedVideoPath: ''
+      },
+      order: {
+        createdAt: 'DESC'
+      },
+      take: 1
+    });
+
+    if (assets.length > 0) {
+      const asset = assets[0];
+      await this.videoEncodingQueue.add('to-mp4', { asset }, { jobId: asset.id },)
+    }
+  }
+}