Merge pull request #9 from KainosSoftwareLtd/VEL-1258-pagination-performance-issues

introduce totalMethod param to get fast but approx count

Author: szymonp-kainos

src/fhir/query_string.coffee

@@ -125,7 +125,10 @@ specials =
     query.sort ||= []
     query.sort.push ['$param', key, '']
     query
-  format:  (query, left, right)->
+  totalMethod: (query, left, right)->
+    query.total_method = right;
+    query
+  format: (query, left, right)->
     query
 
 grouping = (acc, expr)->

src/fhir/search.litcoffee

@@ -32,7 +32,6 @@ so the code split as much as possible  into small modules
     lang = require('../lang')
     compat = require('../compat')
 
-
 For every search type we have dedicated module,
 with indexing and building search expression implementation.
 
@@ -101,27 +100,36 @@ Then we are returning  resulting Bundle.
 
       resource_rows = utils.exec(plv8, honey)
       resources = resource_rows.map((x)-> compat.parse(plv8, x.resource))
-      count = utils.exec(plv8, countize_query(honey))[0].count
+
+      should_include_total = (expr.total_method != "no")
+
+      if should_include_total
+        count = get_count(plv8, honey, expr)
 
       base_url = "#{query.resourceType}/#{query.queryString}"
 
       if expr.summary or expr.elements
         resources = mask_resources(plv8, expr, idx_db, resources)
 
-      if expr.include && count > 0
+      if expr.include && (count > 0 || !should_include_total)
         includes = search_include.load_includes(plv8, expr.include, resources)
         resources = resources.concat(includes)
 
-      if expr.revinclude && count > 0
+      if expr.revinclude && (count > 0 || !should_include_total)
         includes = search_include.load_revincludes(plv8, expr.revinclude, resources)
         resources = resources.concat(includes)
 
+      if should_include_total
+        resourceType: 'Bundle'
+        type: 'searchset'
+        total: count
+        link: helpers.search_links(query, expr, count)
+        entry: resources.map(to_entry)
+      else
+        resourceType: 'Bundle'
+        type: 'searchset'
+        entry: resources.map(to_entry)
 
-      resourceType: 'Bundle'
-      type: 'searchset'
-      total: count
-      link: helpers.search_links(query, expr, count)
-      entry: resources.map(to_entry)
 
 Helper function to convert resource into entry bundle:
 TODO: add links
@@ -388,6 +396,27 @@ we just strip limit, offset, order and rewrite select clause:
       q.select = [':count(*) as count']
       q
 
+    get_count = (plv8, honey, query_obj) ->
+      if !query_obj.total_method || query_obj.total_method is "std"
+        query_obj.total_method = "std"
+
+        utils.exec(plv8, countize_query(honey))[0].count
+      else if query_obj.total_method is "improved"
+        sql_query= sql(honey)
+        query = sql_query[0].replace /\$(\d+)/g, (match, number) ->
+            if typeof sql_query[number] is "string"
+              return '\''+sql_query[number]+'\''
+            else if typeof sql_query[number] isnt 'undefined'
+              return sql_query[number]
+            else
+              return match
+        query = query.replace /LIMIT \d+/, ""
+        query = query.replace /\'/g, '\'\''
+        query = "SELECT count_estimate('#{query}');"
+        tmp = plv8.execute(query)
+        tmp[0].count_estimate
+      else
+        throw new Error("Invalid value of totalMethod. only 'std', 'improved' and 'no' allowed.")
 
 We cache FHIR meta-data index per connection using plv8 object:
 

utils/generate_migrations.coffee

@@ -77,6 +77,22 @@ CREATE AGGREGATE FIRST (
   basetype = anyelement,
   stype    = anyelement
 );
+
+-- Create function to estimate rows of a query. It's used to improve performance of search function.
+CREATE OR REPLACE FUNCTION count_estimate(query text) RETURNS INTEGER AS
+$func$
+DECLARE
+    rec   record;
+    ROWS  INTEGER;
+BEGIN
+    FOR rec IN EXECUTE 'EXPLAIN ' || query LOOP
+        ROWS := SUBSTRING(rec."QUERY PLAN" FROM ' rows=([[:digit:]]+)');
+        EXIT WHEN ROWS IS NOT NULL;
+    END LOOP;
+
+    RETURN ROWS;
+END
+$func$ LANGUAGE plpgsql;
 """
 
 is_first = true