Much smarter merge: cleanup items that only exist in a previous result

lib/rspec/openapi/hooks.rb

@@ -4,6 +4,7 @@
 require 'rspec/openapi/schema_builder'
 require 'rspec/openapi/schema_file'
 require 'rspec/openapi/schema_merger'
+require 'rspec/openapi/schema_cleaner'
 
 path_records = Hash.new { |h, k| h[k] = [] }
 error_records = {}
@@ -23,13 +24,17 @@
       schema = RSpec::OpenAPI::DefaultSchema.build(title)
       schema[:info].merge!(RSpec::OpenAPI.info)
       RSpec::OpenAPI::SchemaMerger.merge!(spec, schema)
+      new_from_zero = {}
       records.each do |record|
         begin
-          RSpec::OpenAPI::SchemaMerger.merge!(spec, RSpec::OpenAPI::SchemaBuilder.build(record))
+          record_schema = RSpec::OpenAPI::SchemaBuilder.build(record)
+          RSpec::OpenAPI::SchemaMerger.merge!(spec, record_schema)
+          RSpec::OpenAPI::SchemaMerger.merge!(new_from_zero, record_schema)
         rescue StandardError, NotImplementedError => e # e.g. SchemaBuilder raises a NotImplementedError
           error_records[e] = record # Avoid failing the build
         end
       end
+      RSpec::OpenAPI::SchemaCleaner.cleanup!(spec, new_from_zero)
     end
   end
   if error_records.any?

lib/rspec/openapi/schema_cleaner.rb

@@ -0,0 +1,101 @@
+# For Ruby 3.0+
+require 'set'
+
+class << RSpec::OpenAPI::SchemaCleaner = Object.new
+  # Cleanup specific elements that exists in the base but not in the spec
+  #
+  # @param [Hash] base
+  # @param [Hash] spec
+  def cleanup!(base, spec)
+    # cleanup URLs
+    cleanup_hash!(base, spec, 'paths.*')
+
+    # cleanup HTTP methods
+    cleanup_hash!(base, spec, 'paths.*.*')
+
+    # cleanup parameters
+    cleanup_array!(base, spec, 'paths.*.*.parameters', %w[name in])
+
+    # cleanup requestBody
+    cleanup_hash!(base, spec, 'paths.*.*.requestBody.content.application/json.schema.properties.*')
+    cleanup_hash!(base, spec, 'paths.*.*.requestBody.content.application/json.example.*')
+
+    # cleanup responses
+    cleanup_hash!(base, spec, 'paths.*.*.responses.*.content.application/json.schema.properties.*')
+    cleanup_hash!(base, spec, 'paths.*.*.responses.*.content.application/json.example.*')
+    base
+  end
+
+  private
+
+  def paths_to_all_fields(obj)
+    case obj
+    when Hash
+      obj.each.flat_map do |k,v|
+        k = k.to_s
+        [[k]] + paths_to_all_fields(v).map { |x| [k, *x] }
+      end
+    else
+      []
+    end
+  end
+
+  def matched_paths(obj, selector)
+    selector_parts = selector.split('.').map(&:to_s)
+    selectors = paths_to_all_fields(obj).select do |key_parts|
+      key_parts.size == selector_parts.size && key_parts.zip(selector_parts).all? do |kp, sp|
+        kp == sp || (sp == '*' && kp != nil)
+      end
+    end
+    selectors
+  end
+
+  def cleanup_array!(base, spec, selector, fields_for_identity = [])
+    marshal = lambda do |obj|
+      Marshal.dump(slice(obj, fields_for_identity))
+    end
+
+    matched_paths(base, selector).each do |paths|
+      target_array = base.dig(*paths)
+      spec_array = spec.dig(*paths)
+      unless target_array.is_a?(Array) && spec_array.is_a?(Array)
+        next
+      end
+      spec_identities = Set.new(spec_array.map(&marshal))
+      target_array.select! { |e| spec_identities.include?(marshal.call(e)) }
+      target_array
+        .sort_by! { |param| [param['__marker'], *fields_for_identity.map {|f| param[f] }].join('-') }
+        .each { |param| param.delete('__marker') }
+      # Keep the last duplicate with largest __marker, to produce the result stably
+      deduplicated = (target_array.reverse.uniq do |param|
+        slice(param, fields_for_identity)
+      end).reverse
+      target_array.replace(deduplicated)
+    end
+    base
+  end
+
+  def cleanup_hash!(base, spec, selector)
+    matched_paths(base, selector).each do |paths|
+      exist_in_base = !base.dig(*paths).nil?
+      not_in_spec = spec.dig(*paths).nil?
+      if exist_in_base && not_in_spec
+        if paths.size == 1
+          base.delete(paths.last)
+        else
+          parent_node = base.dig(*paths[0..-2])
+          parent_node.delete(paths.last)
+        end
+      end
+    end
+    base
+  end
+
+  def slice(obj, fields_for_identity)
+    if fields_for_identity.any?
+      obj.slice(*fields_for_identity)
+    else
+      obj
+    end
+  end
+end

spec/rails/doc/openapi.json

@@ -106,33 +106,42 @@
         },
         "parameters": [
           {
-            "name": "filter[price]",
+            "name": "X-Authorization-Token",
+            "in": "header",
+            "required": true,
+            "schema": {
+              "type": "string"
+            },
+            "example": "token"
+          },
+          {
+            "name": "filter[name]",
             "in": "query",
             "schema": {
               "type": "object",
               "properties": {
-                "price": {
+                "name": {
                   "type": "string"
                 }
               }
             },
             "example": {
-              "price": "0"
+              "name": "Example Table"
             }
           },
           {
-            "name": "filter[name]",
+            "name": "filter[price]",
             "in": "query",
             "schema": {
               "type": "object",
               "properties": {
-                "name": {
+                "price": {
                   "type": "string"
                 }
               }
             },
             "example": {
-              "name": "Example Table"
+              "price": "0"
             }
           },
           {
@@ -150,15 +159,6 @@
               "type": "integer"
             },
             "example": 10
-          },
-          {
-            "name": "X-Authorization-Token",
-            "in": "header",
-            "required": true,
-            "schema": {
-              "type": "string"
-            },
-            "example": "token"
           }
         ]
       },
@@ -647,4 +647,4 @@
       }
     }
   }
-}
+}

spec/rails/doc/openapi.yaml

@@ -20,24 +20,30 @@ paths:
       tags:
       - Table
       parameters:
-      - name: filter[price]
+      - name: X-Authorization-Token
+        in: header
+        required: true
+        schema:
+          type: string
+        example: token
+      - name: filter[name]
         in: query
         schema:
           type: object
           properties:
-            price:
+            name:
               type: string
         example:
-          price: '0'
-      - name: filter[name]
+          name: Example Table
+      - name: filter[price]
         in: query
         schema:
           type: object
           properties:
-            name:
+            price:
               type: string
         example:
-          name: Example Table
+          price: '0'
       - name: page
         in: query
         schema:
@@ -48,12 +54,6 @@ paths:
         schema:
           type: integer
         example: 10
-      - name: X-Authorization-Token
-        in: header
-        required: true
-        schema:
-          type: string
-        example: token
       responses:
         '200':
           description: with different deep query parameters

spec/rails/doc/smart/expected.yaml

@@ -20,40 +20,22 @@ paths:
       tags:
       - Table
       parameters:
-      - name: page
-        in: query
-        schema:
-          type: integer
-        example: 42
-      - name: per
-        in: query
-        schema:
-          type: integer
-        example: 10
       - name: X-Authorization-Token
         in: header
         required: true
         schema:
           type: string
         example: token
-      - name: filter[name]
+      - name: page
         in: query
         schema:
-          type: object
-          properties:
-            name:
-              type: string
-        example:
-          name: Example Table
-      - name: filter[price]
+          type: integer
+        example: 42
+      - name: per
         in: query
         schema:
-          type: object
-          properties:
-            price:
-              type: string
-        example:
-          price: '0'
+          type: integer
+        example: 10
       responses:
         '200':
           description: with flat query parameters
@@ -83,49 +65,8 @@ paths:
                 properties:
                   message:
                     type: string
-                  no_such_field:
-                    type: string
-                    example: This field does not exist in rspec
               example:
                 message: Unauthorized
-    post:
-      summary: create
-      tags:
-      - Table
-      requestBody:
-        content:
-          application/json:
-            schema:
-              type: object
-              properties:
-                name:
-                  type: string
-                description:
-                  type: string
-                database_id:
-                  type: integer
-            example:
-              name: k0kubun
-              description: description
-              database_id: 2
-      responses:
-        '201':
-          description: returns a table
-          content:
-            application/json:
-              schema:
-                "$ref": "#/components/schemas/Table"
-              example:
-                id: 1
-                name: access
-                description: logs
-                database:
-                  id: 2
-                  name: production
-                null_sample:
-                storage_size: 12.3
-                created_at: '2020-07-17T00:00:00+00:00'
-                updated_at: '2020-07-17T00:00:00+00:00'
   "/tables/{id}":
     get:
       summary: show

spec/rails/doc/smart/openapi.yaml

@@ -14,6 +14,19 @@ info:
 servers:
 - url: http://localhost:3000
 paths:
+  "/no-such-path":
+    get:
+      summary: no such api
+      parameters: []
+      responses:
+        '200':
+          description: dummy
+          content:
+            application/json:
+              schema:
+                type: number
+              example:
+                - 1
   "/tables":
     get:
       summary: index
@@ -104,10 +117,13 @@ paths:
                   type: string
                 database_id:
                   type: integer
+                no_such_field_request:
+                  type: boolean
             example:
               name: k0kubun
               description: description
               database_id: 2
+              no_such_field_request: true
       responses:
         '201':
           description: returns a table
@@ -164,6 +180,8 @@ paths:
                   storage_size:
                     type: number
                     format: float
+                  no_such_field_response:
+                    type: boolean
                   created_at:
                     type: string
                   updated_at:
@@ -172,6 +190,7 @@ paths:
                 id: 1
                 name: access
                 description: logs
+                no_such_field_response: true
                 database:
                   id: 2
                   name: production