sqlite optimizations

CONTRIBUTING.md

@@ -130,7 +130,8 @@ When reporting issues, please include:
 ## Development Tips
 
 1. **Testing with large C# projects**: The plugin is designed for large codebases. Test your changes with substantial
-C# projects like the [.NET Reference Source](https://github.com/microsoft/referencesource)
+C# projects like the [.NET Reference Source](https://github.com/microsoft/referencesource) or [azure-sdk-for-net](https://github.com/Azure/azure-sdk-for-net)
+latter is especially heave, around 100k source code files.
 
 2. **Database debugging**: You can inspect the SQLite database directly in lua code:
    ```lua

lua/hopcsharp/database/init.lua

@@ -58,14 +58,26 @@ M.__get_db = function()
     -- add empty namespace
     _db:insert('namespaces', { name = 'n\\a' })
 
-    -- some dumb performance optimizations
-    _db:execute('pragma synchronous = OFF')
-    _db:execute('pragma cache_size = -32768')
-    _db:execute('pragma journal_mode = OFF')
+    -- dumb performance optimizations
+    _db:execute('PRAGMA cache_size = -131072') -- 128MB cache
+    _db:execute('PRAGMA journal_mode = OFF')
+    _db:execute('PRAGMA synchronous = OFF')
+    _db:execute('PRAGMA page_size = 8192')
+    _db:execute('PRAGMA mmap_size = 1073741824') -- 1GB mmap
+    _db:execute('PRAGMA temp_store = MEMORY')
+    _db:execute('PRAGMA optimize')
 
     return _db
 end
 
+M.__create_indexes = function()
+    local db = M.__get_db()
+    db:eval('CREATE INDEX IF NOT EXISTS idx_definitions_name_type ON definitions(name, type);')
+    db:eval('CREATE INDEX IF NOT EXISTS idx_inheritance_name ON inheritance(name);')
+    db:eval('CREATE INDEX IF NOT EXISTS idx_inheritance_base ON inheritance(base);')
+    db:eval('CREATE INDEX IF NOT EXISTS idx_reference_name_type ON reference(name, type);')
+end
+
 M.__drop_db = function()
     local db = M.__get_db()
     db:eval('delete from definitions')

lua/hopcsharp/database/query.lua

@@ -1,40 +1,50 @@
 local M = {}
 
-M.get_definition_by_name = [[
-    SELECT
-        d.name,
-        f.path,
-        d.row,
-        d.column,
-        d.type,
-        n.name AS namespace
-    FROM definitions d
-    JOIN files f on f.id = d.path_id
-    JOIN namespaces n on n.id = d.namespace_id
-    WHERE d.name = :name OR d.name LIKE :name || '<%>'
-    ORDER BY
-        n.name ASC,
-        f.path ASC
-]]
+M.get_definition_by_name = function(name)
+    local query = [[
+        SELECT
+            d.name,
+            f.path,
+            d.row,
+            d.column,
+            d.type,
+            n.name AS namespace
+        FROM definitions d
+        JOIN files f on f.id = d.path_id
+        JOIN namespaces n on n.id = d.namespace_id
+        WHERE d.name = '%s' OR d.name GLOB '%s'
+        ORDER BY
+            n.name ASC,
+            f.path ASC
+    ]]
 
-M.get_definition_by_name_and_type = [[
-    SELECT
-        d.name,
-        f.path,
-        d.row,
-        d.column,
-        d.type,
-        n.name AS namespace
-    FROM definitions d
-    JOIN files f on f.id = d.path_id
-    JOIN namespaces n on n.id = d.namespace_id
-    WHERE (d.name = :name OR d.name = :name || 'Attribute' OR d.name LIKE :name || '<%>')
-        AND d.type = :type
-    ORDER BY
-        d.name ASC,
-        n.name ASC,
-        f.path ASC
-]]
+    -- using GLOB to make use of indexes
+    return string.format(query, name, name .. '<*>')
+end
+
+M.get_definition_by_name_and_type = function(name, type)
+    local query = [[
+        SELECT
+            d.name,
+            f.path,
+            d.row,
+            d.column,
+            d.type,
+            n.name AS namespace
+        FROM definitions d
+        JOIN files f on f.id = d.path_id
+        JOIN namespaces n on n.id = d.namespace_id
+        WHERE (d.name = '%s' OR d.name GLOB '%s'  OR d.name GLOB '%s')
+            AND d.type = %s
+        ORDER BY
+            d.name ASC,
+            n.name ASC,
+            f.path ASC
+    ]]
+
+    -- using GLOB to make use of indexes
+    return string.format(query, name, name .. 'Attribute', name .. '<*>', type)
+end
 
 M.get_all_definitions = [[
     SELECT
@@ -71,7 +81,8 @@ M.get_definition_by_type = [[
         f.path ASC
 ]]
 
-M.get_definition_by_name_and_usings = function(usings)
+-- TODO think of by name, type and usings?
+M.get_definition_by_name_and_usings = function(name, usings)
     local query = [[
         SELECT
             d.name,
@@ -83,8 +94,7 @@ M.get_definition_by_name_and_usings = function(usings)
         FROM definitions d
         JOIN files f on f.id = d.path_id
         JOIN namespaces n on n.id = d.namespace_id
-        WHERE (d.name = :name OR d.name LIKE :name || '<%%>')
-            AND n.name IN (%s)
+        WHERE (d.name = '%s' OR d.name GLOB '%s') AND n.name IN (%s)
         ORDER BY
             n.name ASC,
             f.path ASC
@@ -94,7 +104,7 @@ M.get_definition_by_name_and_usings = function(usings)
         usings[i] = '"' .. using .. '"'
     end
 
-    return string.format(query, table.concat(usings, ','))
+    return string.format(query, name, name .. '<*>', table.concat(usings, ','))
 end
 
 M.get_attributes = [[
@@ -118,26 +128,30 @@ M.get_attributes = [[
 -- have to add distinct here to avoid listing
 -- classes that has the same name as :name but are not
 -- implementing anything
-M.get_implementations_by_name = [[
-    SELECT DISTINCT
-        d.name,
-        f.path,
-        d.row,
-        d.column,
-        d.type,
-        n.name AS namespace
-    FROM inheritance i
-    JOIN definitions d on d.name = i.name
-    JOIN files f on f.id = d.path_id
-    JOIN namespaces n on n.id = d.namespace_id
-    WHERE (i.base = :name OR i.base LIKE :name || '<%>')
-        AND d.type <> 7 -- filter constructors
-        AND d.type <> 6 -- filter methods
-    ORDER BY
-        d.name ASC,
-        n.name ASC,
-        f.path ASC
-]]
+M.get_implementations_by_name = function(name)
+    local query = [[
+        SELECT DISTINCT
+            d.name,
+            f.path,
+            d.row,
+            d.column,
+            d.type,
+            n.name AS namespace
+        FROM inheritance i
+        JOIN definitions d on d.name = i.name
+        JOIN files f on f.id = d.path_id
+        JOIN namespaces n on n.id = d.namespace_id
+        WHERE (i.base = '%s' OR i.base GLOB '%s')
+            AND d.type <> 7 -- filter constructors
+            AND d.type <> 6 -- filter methods
+        ORDER BY
+            d.name ASC,
+            n.name ASC,
+            f.path ASC
+    ]]
+
+    return string.format(query, name, name .. '<*>')
+end
 
 -- this is kind of a hack here
 -- we get implementaiton defentions from definitions table
@@ -191,41 +205,47 @@ M.get_all_child_types = [[
     SELECT DISTINCT name, base FROM children WHERE base <> name;
 ]]
 
-M.get_reference_by_name = [[
-    SELECT
-        r.name,
-        f.path,
-        r.row,
-        r.column,
-        r.type,
-        n.name AS namespace
-    FROM reference r
-    JOIN files f on f.id = r.path_id
-    JOIN namespaces n on n.id = r.namespace_id
-    WHERE (r.name = :name OR r.name LIKE :name || '<%>' OR r.name || 'Attribute' = :name)
-    ORDER BY
-        r.name ASC,
-        n.name ASC,
-        f.path ASC
-]]
+M.get_reference_by_name = function(name)
+    local query = [[
+        SELECT
+            r.name,
+            f.path,
+            r.row,
+            r.column,
+            r.type,
+            n.name AS namespace
+        FROM reference r
+        JOIN files f on f.id = r.path_id
+        JOIN namespaces n on n.id = r.namespace_id
+        WHERE (r.name = '%s' OR r.name GLOB '%s' OR r.name || 'Attribute' = '%s')
+        ORDER BY
+            r.name ASC,
+            n.name ASC,
+            f.path ASC
+    ]]
+    return string.format(query, name, name .. '<*>', name)
+end
 
-M.get_reference_by_name_and_type = [[
-    SELECT
-        r.name,
-        f.path,
-        r.row,
-        r.column,
-        r.type,
-        n.name AS namespace
-    FROM reference r
-    JOIN files f on f.id = r.path_id
-    JOIN namespaces n on n.id = r.namespace_id
-    WHERE (r.name = :name OR r.name LIKE :name || '<%>')
-        AND r.type = :type
-    ORDER BY
-        r.name ASC,
-        n.name ASC,
-        f.path ASC
-]]
+M.get_reference_by_name_and_type = function(name, type)
+    local query = [[
+        SELECT
+            r.name,
+            f.path,
+            r.row,
+            r.column,
+            r.type,
+            n.name AS namespace
+        FROM reference r
+        JOIN files f on f.id = r.path_id
+        JOIN namespaces n on n.id = r.namespace_id
+        WHERE (r.name = '%s' OR r.name GLOB '%s') AND r.type = %s
+        ORDER BY
+            r.name ASC,
+            n.name ASC,
+            f.path ASC
+    ]]
+
+    return string.format(query, name, name .. '<*>', type)
+end
 
 return M

lua/hopcsharp/database/utils.lua

@@ -76,23 +76,23 @@ M.get_reference_type_name = function(type)
     end
 
     if type == M.reference_types.METHOD_INVOCATION then
-        return 'method invocation'
+        return 'method'
     end
 
     if type == M.reference_types.OBJECT_CREATION then
-        return 'object creation'
+        return 'object'
     end
 
     if type == M.reference_types.VARIABLE_DECLARATION then
-        return 'variable declaration'
+        return 'variable'
     end
 
     if type == M.reference_types.TYPE_ARGUMENT then
-        return 'type argument'
+        return 'type'
     end
 
     if type == M.reference_types.TYPEOF_EXPRESSION then
-        return 'typeof reference'
+        return 'type'
     end
 end
 

lua/hopcsharp/hierarchy/utils.lua

@@ -11,10 +11,7 @@ local function find_first_class(type_relations, type_name)
     for _, entry in ipairs(type_relations) do
         if entry['name'] == type_name then
             -- if current entry base type is a class -> bingo!
-            local items = db:eval(
-                query.get_definition_by_name_and_type,
-                { name = entry['base'], type = database_utils.types.CLASS }
-            )
+            local items = db:eval(query.get_definition_by_name_and_type(entry['base'], database_utils.types.CLASS))
 
             -- if we have found any class in result
             if type(items) == 'table' then

lua/hopcsharp/hop/init.lua

@@ -1,4 +1,5 @@
-local utils = require('hopcsharp.hop.utils')
+local utils = require('hopcsharp.utils')
+local hop_utils = require('hopcsharp.hop.utils')
 local dbutils = require('hopcsharp.database.utils')
 
 local definition_providers = require('hopcsharp.hop.providers.definition')
@@ -7,19 +8,35 @@ local implementation_providers = require('hopcsharp.hop.providers.implementation
 
 local M = {}
 
-local function populate_quickfix(entries, jump_on_quickfix, type_converter)
-    local qflist = {}
+local stop_callback = nil
 
-    for _, entry in ipairs(entries) do
-        table.insert(qflist, {
-            filename = entry.path,
-            lnum = entry.row + 1,
-            col = entry.col,
-            text = (entry.namespace or '') .. ' | ' .. type_converter(entry.type) .. ' | ' .. entry.name,
-        })
+local function populate_quickfix(entries, jump_on_quickfix, type_converter)
+    -- stop previous quickfix population
+    -- won't work 100% but it much better
+    -- rathen that nothing
+    if stop_callback then
+        stop_callback()
+        stop_callback = nil
     end
 
-    vim.fn.setqflist(qflist, 'r')
+    -- remove previous quickfix entries
+    vim.fn.setqflist({}, 'r')
+
+    utils.__scheduled_iteration(entries, function(i, item, _, stop)
+        if i == 1 then
+            stop_callback = stop
+        end
+
+        vim.fn.setqflist({
+            {
+                filename = item.path,
+                lnum = item.row + 1,
+                col = item.col,
+                text = string.format('%-15s | %s', type_converter(item.type), item.namespace or ''),
+            },
+        }, 'a')
+    end)
+
     vim.cmd([[ :copen ]])
 
     if jump_on_quickfix then
@@ -75,7 +92,7 @@ M.__hop_to = function(hop_providers, config)
 
                 -- immediate jump if there is only one case
                 if #filtered_items == 1 then
-                    utils.__hop(filtered_items[1].path, filtered_items[1].row + 1, filtered_items[1].column)
+                    hop_utils.__hop(filtered_items[1].path, filtered_items[1].row + 1, filtered_items[1].column)
                     return
                 end