grpc poc working on k8s, using envoy as a sidecar in catalog api

eiximenis · eiximenis · commit 591086956d80 · 2019-06-27T16:16:11.000+02:00
diff --git a/docker-compose.override.yml b/docker-compose.override.yml
@@ -89,6 +89,8 @@ services:
       - AzureStorageEnabled=False
       - ApplicationInsights__InstrumentationKey=${INSTRUMENTATION_KEY}
       - OrchestratorType=${ORCHESTRATOR_TYPE}
+      - GRPC_PORT=81
+      - PORT=80
     ports:
       - "5101:80"
       - "9101:81"
diff --git a/docker-compose.yml b/docker-compose.yml
@@ -240,7 +240,4 @@ services:
       - webhooks.api    
 
   envoy:
-    image: envoy:v1
-    build:
-      context: src/ApiGateways/Envoy
-      dockerfile: Dockerfile
+    image: envoyproxy/envoy
diff --git a/k8s/helm/apigwws/configuration-web-shopping.json b/k8s/helm/apigwws/configuration-web-shopping.json
@@ -6,12 +6,24 @@
         "DownstreamHostAndPorts": [
           {
             "Host": "catalog",
-            "Port": 80
+            "Port": 5000
           }
         ],
         "UpstreamPathTemplate": "/api/{version}/c/{everything}",
         "UpstreamHttpMethod": [ "GET" ]
       },
+      {
+        "DownstreamPathTemplate": "/api/{version}/{everything}",
+        "DownstreamScheme": "http",
+        "DownstreamHostAndPorts": [
+          {
+            "Host": "catalog",
+            "Port": 80
+          }
+        ],
+        "UpstreamPathTemplate": "/grpc/{version}/c/{everything}",
+        "UpstreamHttpMethod": [ "GET" ]
+      },      
       {
         "DownstreamPathTemplate": "/api/{version}/{everything}",
         "DownstreamScheme": "http",
@@ -102,7 +114,7 @@
         "DownstreamHostAndPorts": [
           {
             "Host": "catalog",
-            "Port": 80
+            "Port": 5000
           }
         ],
         "UpstreamPathTemplate": "/catalog-api/{everything}",
diff --git a/k8s/helm/catalog-api/envoycfg/_catalog.proto-descriptor.pb b/k8s/helm/catalog-api/envoycfg/_catalog.proto-descriptor.pb
diff --git a/k8s/helm/catalog-api/envoycfg/_envoy.yaml b/k8s/helm/catalog-api/envoycfg/_envoy.yaml
@@ -0,0 +1,56 @@
+admin:
+  access_log_path: /tmp/admin_access.log
+  address:
+    socket_address: { address: 0.0.0.0, port_value: 9901 }
+
+static_resources:
+  listeners:
+  - name: listener1
+    address:
+      socket_address: { address: 0.0.0.0, port_value: 51051 }
+    filter_chains:
+    - filters:
+      - name: envoy.http_connection_manager
+        typed_config:
+          "@type": type.googleapis.com/envoy.config.filter.network.http_connection_manager.v2.HttpConnectionManager
+          stat_prefix: grpc_json
+          codec_type: AUTO
+          route_config:
+            name: local_route
+            virtual_hosts:
+            - name: local_service
+              domains: ["*"]
+              routes:
+              - match: { prefix: "/" }
+                route: { cluster: grpc, timeout: { seconds: 60 } }
+          http_filters:
+          - name: envoy.grpc_json_transcoder
+            config:
+              proto_descriptor: "/etc/envoy/catalog.proto-descriptor.pb"
+              services: ["CatalogApi.Catalog"]
+              print_options:
+                add_whitespace: true
+                always_print_primitive_fields: true
+                always_print_enums_as_ints: false
+                preserve_proto_field_names: false
+          - name: envoy.router
+
+  clusters:
+  - name: grpc
+    connect_timeout: 1.25s
+    type: logical_dns
+    lb_policy: round_robin
+    dns_lookup_family: V4_ONLY
+    http2_protocol_options: {}
+    load_assignment:
+      cluster_name: grpc
+      endpoints:
+      - lb_endpoints:
+        - endpoint:
+            address:
+              socket_address:
+                # WARNING: "docker.for.mac.localhost" has been deprecated from Docker v18.03.0.
+                # If you're running an older version of Docker, please use "docker.for.mac.localhost" instead.
+                # Reference: https://docs.docker.com/docker-for-mac/release-notes/#docker-community-edition-18030-ce-mac59-2018-03-26
+                address: localhost
+                port_value: 5001
diff --git a/k8s/helm/catalog-api/templates/deployment.yaml b/k8s/helm/catalog-api/templates/deployment.yaml
@@ -73,10 +73,23 @@ spec:
           {{- end }}
           ports:
             - name: http
-              containerPort: 80
+              containerPort: 5000
               protocol: TCP
+            - name: grpc
+              containerPort: 5001
+              protocol: TCP              
           resources:
 {{ toYaml .Values.resources | indent 12 }}
+        - name: envoy-proxy
+          image: envoyproxy/envoy
+          imagePullPolicy: IfNotPresent
+          ports:
+            - name: envoy
+              containerPort: 51051
+              protocol: TCP
+          volumeMounts:
+            - name: envoy-config
+              mountPath: /etc/envoy
     {{- with .Values.nodeSelector }}
       nodeSelector:
 {{ toYaml . | indent 8 }}
@@ -89,4 +102,13 @@ spec:
       tolerations:
 {{ toYaml . | indent 8 }}
     {{- end }}
+      volumes:
+        - name: envoy-config
+          configMap:
+            name: envoy-{{ $name }}
+            items:
+              - key: _envoy.yaml
+                path: envoy.yaml
+              - key: _catalog.proto-descriptor.pb
+                path: catalog.proto-descriptor.pb
 
diff --git a/k8s/helm/catalog-api/templates/envoy-cm.yaml b/k8s/helm/catalog-api/templates/envoy-cm.yaml
@@ -0,0 +1,16 @@
+{{- $name := include "catalog-api.fullname" . -}}
+
+apiVersion: v1
+kind: ConfigMap
+metadata:
+  name: "envoy-{{ $name }}"
+  labels:
+    app: {{ template "catalog-api.name" . }}
+    chart: {{ template "catalog-api.chart" .}}
+    release: {{ .Release.Name }}
+    heritage: {{ .Release.Service }}
+data:
+{{ (.Files.Glob "envoycfg/*.yaml").AsConfig | indent 2 }}  
+binaryData:
+  _catalog.proto-descriptor.pb: |-
+    {{ .Files.Get "envoycfg/_catalog.proto-descriptor.pb" | b64enc -}} 
diff --git a/k8s/helm/catalog-api/templates/service.yaml b/k8s/helm/catalog-api/templates/service.yaml
@@ -14,6 +14,14 @@ spec:
       targetPort: http
       protocol: TCP
       name: http
+    - port: {{ .Values.service.grpc }}
+      targetPort: grpc
+      protocol: TCP
+      name: grpc     
+    - port: {{ .Values.service.envoy }}
+      targetPort: envoy
+      protocol: TCP
+      name: envoy      
   selector:
     app: {{ template "catalog-api.name" . }}
     release: {{ .Release.Name }}
diff --git a/k8s/helm/catalog-api/values.yaml b/k8s/helm/catalog-api/values.yaml
@@ -9,7 +9,9 @@ image:
 
 service:
   type: ClusterIP
-  port: 80
+  port: 5000
+  grpc: 5001
+  envoy: 80
 
 
 resources: {}
@@ -44,16 +46,20 @@ env:
       value: Development
     - name: OrchestratorType
       value: 'K8S'
+    - name: PORT
+      value: "5000"
+    - name: GRPC_PORT
+      value: "5001"
 probes:
   liveness:
     path: /liveness
     initialDelaySeconds: 10
     periodSeconds: 15
-    port: 80
+    port: 5000
   readiness:
     path: /hc
     timeoutSeconds: 5
     initialDelaySeconds: 90
     periodSeconds: 60
-    port: 80
+    port: 5000
 
diff --git a/src/ApiGateways/Web.Bff.Shopping/aggregator/Config/UrlsConfig.cs b/src/ApiGateways/Web.Bff.Shopping/aggregator/Config/UrlsConfig.cs
@@ -9,8 +9,10 @@ public class UrlsConfig
     {
         public class CatalogOperations
         {
+            // grpc call under REST must go trough port 80
             public static string GetItemById(int id) => $"/api/v1/catalog/items/{id}";
-            public static string GetItemsById(IEnumerable<int> ids) => $"/api/v1/catalog/items?ids={string.Join(',', ids)}";
+            // REST call standard must go through port 5000
+            public static string GetItemsById(IEnumerable<int> ids) => $":5000/api/v1/catalog/items?ids={string.Join(',', ids)}";
         }
 
         public class BasketOperations
diff --git a/src/ApiGateways/Web.Bff.Shopping/aggregator/Services/CatalogService.cs b/src/ApiGateways/Web.Bff.Shopping/aggregator/Services/CatalogService.cs
@@ -24,7 +24,8 @@ public CatalogService(HttpClient httpClient, ILogger<CatalogService> logger, IOp
 
         public async Task<CatalogItem> GetCatalogItemAsync(int id)
         {
-            var stringContent = await _httpClient.GetStringAsync(_urls.Catalog + UrlsConfig.CatalogOperations.GetItemById(id));
+            var uri=_urls.Catalog + UrlsConfig.CatalogOperations.GetItemById(id);
+            var stringContent = await _httpClient.GetStringAsync(uri);
 
             return JsonConvert.DeserializeObject<CatalogItem>(stringContent);
         }
diff --git a/src/Services/Catalog/Catalog.API/Grpc/CatalogService.cs b/src/Services/Catalog/Catalog.API/Grpc/CatalogService.cs
@@ -8,6 +8,7 @@
 using Microsoft.eShopOnContainers.Services.Catalog.API;
 using Microsoft.eShopOnContainers.Services.Catalog.API.Infrastructure;
 using Microsoft.eShopOnContainers.Services.Catalog.API.Model;
+using Microsoft.Extensions.Logging;
 using Microsoft.Extensions.Options;
 using static CatalogApi.Catalog;
 
@@ -17,15 +18,17 @@ public class CatalogService : CatalogBase
     {
         private readonly CatalogContext _catalogContext;
         private readonly CatalogSettings _settings;
-        public CatalogService(CatalogContext dbContext, IOptions<CatalogSettings> settings)
+        private readonly ILogger _logger;
+        public CatalogService(CatalogContext dbContext, IOptions<CatalogSettings> settings, ILogger<CatalogService> logger)
         {
             _settings = settings.Value;
             _catalogContext = dbContext ?? throw new ArgumentNullException(nameof(dbContext));
+            _logger = logger;
         }
 
        public override async Task<CatalogItemResponse> GetItemById(CatalogItemRequest request, ServerCallContext context)
         {
-
+            _logger.LogInformation($"Begin grpc call CatalogService.GetItemById for product id {request.Id}");
             if (request.Id <=0)
             {
                 context.Status = new Status(StatusCode.FailedPrecondition, $"Id must be > 0 (received {request.Id})");
diff --git a/src/Services/Catalog/Catalog.API/Program.cs b/src/Services/Catalog/Catalog.API/Program.cs
@@ -75,11 +75,12 @@ private static IHostBuilder CreateHostBuilder(IConfiguration configuration, stri
                     .UseConfiguration(configuration)
                     .ConfigureKestrel(options =>
                     {
-                        options.Listen(IPAddress.Any, 80, listenOptions =>
+                        var ports = GetDefinedPorts(configuration);
+                        options.Listen(IPAddress.Any, ports.httpPort, listenOptions =>
                         {
-                            listenOptions.Protocols = HttpProtocols.Http1;
+                            listenOptions.Protocols = HttpProtocols.Http1AndHttp2;
                         });
-                        options.Listen(IPAddress.Any, 81, listenOptions =>
+                        options.Listen(IPAddress.Any, ports.grpcPort, listenOptions =>
                         {
                             listenOptions.Protocols = HttpProtocols.Http2;
                         });
@@ -107,38 +108,11 @@ private static Serilog.ILogger CreateSerilogLogger(IConfiguration configuration)
                 .CreateLogger();
         }
 
-        private static IEnumerable<(int portNumber, bool https)> GetDefinedPorts(IConfiguration config)
+        private static (int httpPort, int grpcPort) GetDefinedPorts(IConfiguration config)
         {
-            const string https = "https://";
-            const string http = "http://";
-            var defport = config.GetValue("ASPNETCORE_HTTPS_PORT", 0);
-            if (defport != 0)
-            {
-                yield return (defport, true);
-            }
-
-            var urls = config.GetValue<string>("ASPNETCORE_URLS", null)?.Split(new[] { ';' }, StringSplitOptions.RemoveEmptyEntries);
-            if (urls?.Any() == true)
-            {
-                foreach (var urlString in urls)
-                {
-                    var uri = urlString.ToLowerInvariant().Trim();
-                    var isHttps = uri.StartsWith(https);
-                    var isHttp = uri.StartsWith(http);
-                    if (!isHttp && !isHttps)
-                    {
-                        throw new ArgumentException($"Url {uri} must start with https:// or http://");
-                    }
-
-                    uri = uri.Substring(isHttps ? https.Length : http.Length);
-                    var lastdots = uri.LastIndexOf(':');
-                    if (lastdots != -1)
-                    {
-                        var sport = uri.Substring(lastdots + 1);
-                        yield return (int.TryParse(sport, out var nport) ? nport : isHttps ? 443 : 80, isHttps);
-                    }
-                }
-            }
+            var grpcPort = config.GetValue("GRPC_PORT", 5001);
+            var port = config.GetValue("PORT", 80);
+            return (port, grpcPort);
         }
 
         private static IConfiguration GetConfiguration()

Original file line number	Diff line number	Diff line change
`@@ -9,8 +9,10 @@ public class UrlsConfig`
`9`	`9`	`{`
`10`	`10`	`public class CatalogOperations`
`11`	`11`	`{`
	`12`	`+ // grpc call under REST must go trough port 80`
`12`	`13`	`public static string GetItemById(int id) => $"/api/v1/catalog/items/{id}";`
`13`		`- public static string GetItemsById(IEnumerable<int> ids) => $"/api/v1/catalog/items?ids={string.Join(',', ids)}";`
	`14`	`+ // REST call standard must go through port 5000`
	`15`	`+ public static string GetItemsById(IEnumerable<int> ids) => $":5000/api/v1/catalog/items?ids={string.Join(',', ids)}";`
`14`	`16`	`}`
`15`	`17`
`16`	`18`	`public class BasketOperations`