Merge pull request #3463 from vladak/docker_project_less

Docker project less

Author: Vladimir Kotal

docker/README.md

@@ -36,7 +36,8 @@ The mirroring step works by going through all projects and attempting to
 synchronize all its repositories (e.g. it will do `git pull --ff-only` for Git
 repositories).
 
-Projects are enabled in this setup and there is no way how to change that.
+Projects are enabled in this setup by default. See environment variables
+below on how to change that.
 
 ### Indexer logs
 
@@ -52,7 +53,7 @@ is doing, use the `docker logs` command.
 
 ### Tags and versioning
 
-Each OpenGrok release triggers creation of new Docker image. 
+Each OpenGrok release triggers creation of new Docker image.
 
 | Tag      | Note                                                    |
 | -------- |:--------------------------------------------------------|
@@ -78,7 +79,8 @@ The volume mounted to `/opengrok/src` should contain the projects you want to ma
 `INDEXER_OPT` | empty | pass **extra** options to OpenGrok Indexer. The default set of indexer options is: `--remote on -P -H -W`. For example, `-i d:vendor` will remove all the `*/vendor/*` files from the index. You can check the indexer options on https://github.com/oracle/opengrok/wiki/Python-scripts-transition-guide
 `NOMIRROR` | empty | To avoid the mirroring step, set the variable to non-empty value.
 `URL_ROOT` | `/` | Override the sub-URL that OpenGrok should run on.
-`WORKERS` | number of CPUs in the container | number of workers to use for syncing
+`WORKERS` | number of CPUs in the container | number of workers to use for syncing (applies only to setup with projects enabled)
+`AVOID_PROJECTS` | empty | run in project less configuration. Set to non empty value disables projects.
 
 To specify environment variable for `docker run`, use the `-e` option, e.g. `-e SYNC_PERIOD_MINUTES=30`
 

docker/start.py

@@ -60,11 +60,13 @@
 OPENGROK_CONFIG_FILE = os.path.join(OPENGROK_BASE_DIR, "etc",
                                     "configuration.xml")
 OPENGROK_WEBAPPS_DIR = os.path.join(tomcat_root, "webapps")
+OPENGROK_JAR = os.path.join(OPENGROK_LIB_DIR, 'opengrok.jar')
 
 
 def set_url_root(logger, url_root):
     """
     Set URL root and URI based on input
+    :param logger: logger instance
     :param url_root: input
     :return: URI and URL root
     """
@@ -108,6 +110,7 @@ def get_war_name(url_root):
 def deploy(logger, url_root):
     """
     Deploy the web application
+    :param logger: logger instance
     :param url_root: web app URL root
     """
 
@@ -190,6 +193,7 @@ def refresh_projects(logger, uri):
 def save_config(logger, uri, config_path):
     """
     Retrieve configuration from the web app and write it to file.
+    :param logger: logger instance
     :param uri: web app URI
     :param config_path: file path
     """
@@ -218,7 +222,37 @@ def merge_commands_env(commands, env):
     return commands
 
 
-def syncer(logger, loglevel, uri, config_path, sync_period, numworkers, env):
+def indexer_no_projects(logger, uri, config_path, sync_period,
+                        extra_indexer_options):
+    """
+    Project less indexer
+    """
+
+    wait_for_tomcat(logger, uri)
+
+    while True:
+        indexer_options = ['-s', OPENGROK_SRC_ROOT,
+                           '-d', OPENGROK_DATA_ROOT,
+                           '-c', '/usr/local/bin/ctags',
+                           '--remote', 'on',
+                           '-H',
+                           '-W', config_path,
+                           '-U', uri]
+        if extra_indexer_options:
+            logger.debug("Adding extra indexer options: {}".
+                         format(extra_indexer_options))
+            indexer_options.extend(extra_indexer_options.split())
+        indexer = Indexer(indexer_options, logger=logger,
+                          jar=OPENGROK_JAR, doprint=True)
+        indexer.execute()
+
+        sleep_seconds = sync_period * 60
+        logger.info("Sleeping for {} seconds".format(sleep_seconds))
+        time.sleep(sleep_seconds)
+
+
+def project_syncer(logger, loglevel, uri, config_path, sync_period,
+                   numworkers, env):
     """
     Wrapper for running opengrok-sync.
     To be run in a thread/process in the background.
@@ -267,22 +301,25 @@ def syncer(logger, loglevel, uri, config_path, sync_period, numworkers, env):
         time.sleep(sleep_seconds)
 
 
-def create_bare_config(logger):
+def create_bare_config(logger, use_projects=True):
     """
     Create bare configuration file with a few basic settings.
     """
 
     logger.info('Creating bare configuration in {}'.
                 format(OPENGROK_CONFIG_FILE))
-    indexer = Indexer(['-s', OPENGROK_SRC_ROOT,
+    indexer_options = ['-s', OPENGROK_SRC_ROOT,
                        '-d', OPENGROK_DATA_ROOT,
                        '-c', '/usr/local/bin/ctags',
                        '--remote', 'on',
-                       '-P', '-H',
+                       '-H',
                        '-W', OPENGROK_CONFIG_FILE,
-                       '--noIndex'],
-                      jar=os.path.join(OPENGROK_LIB_DIR,
-                                       'opengrok.jar'),
+                       '--noIndex']
+    if use_projects:
+        indexer_options.append('-P')
+
+    indexer = Indexer(indexer_options,
+                      jar=OPENGROK_JAR,
                       logger=logger, doprint=True)
     indexer.execute()
     ret = indexer.getretcode()
@@ -329,39 +366,53 @@ def main():
         setup_redirect_source(logger, url_root)
 
     env = {}
-    if os.environ.get('INDEXER_OPT'):
-        env['OPENGROK_INDEXER_OPTIONAL_ARGS'] = \
-            os.environ.get('INDEXER_OPT')
+    extra_indexer_options = os.environ.get('INDEXER_OPT')
+    if extra_indexer_options:
+        logger.info("extra indexer options: {}".format(extra_indexer_options))
+        env['OPENGROK_INDEXER_OPTIONAL_ARGS'] = extra_indexer_options
     if os.environ.get('NOMIRROR'):
         env['OPENGROK_NO_MIRROR'] = os.environ.get('NOMIRROR')
     logger.debug('Extra environment: {}'.format(env))
 
+    use_projects = True
+    if os.environ.get('AVOID_PROJECTS'):
+        use_projects = False
+
     #
     # Create empty configuration to avoid the non existent file exception
     # in the web app during the first web app startup.
     #
     if not os.path.exists(OPENGROK_CONFIG_FILE) or \
             os.path.getsize(OPENGROK_CONFIG_FILE) == 0:
-        create_bare_config(logger)
+        create_bare_config(logger, use_projects=use_projects)
 
     if sync_period > 0:
-        num_workers = multiprocessing.cpu_count()
-        workers_env = os.environ.get('WORKERS')
-        if workers_env:
-            try:
-                n = int(workers_env)
-                if n > 0:
-                    num_workers = n
-            except ValueError:
-                logger.error("WORKERS is not a number: {}".format(workers_env))
-
-        logger.info('Number of sync workers: {}'.format(num_workers))
+        if use_projects:
+            num_workers = multiprocessing.cpu_count()
+            workers_env = os.environ.get('WORKERS')
+            if workers_env:
+                try:
+                    n = int(workers_env)
+                    if n > 0:
+                        num_workers = n
+                except ValueError:
+                    logger.error("WORKERS is not a number: {}".
+                                 format(workers_env))
+
+            logger.info('Number of sync workers: {}'.format(num_workers))
+
+            worker_function = project_syncer
+            syncer_args = (logger, log_level, uri,
+                           OPENGROK_CONFIG_FILE,
+                           sync_period, num_workers, env)
+        else:
+            worker_function = indexer_no_projects
+            syncer_args = (logger, uri, OPENGROK_CONFIG_FILE, sync_period,
+                           extra_indexer_options)
 
         logger.debug("Starting sync thread")
-        thread = threading.Thread(target=syncer, name="Sync thread",
-                                  args=(logger, log_level, uri,
-                                        OPENGROK_CONFIG_FILE,
-                                        sync_period, num_workers, env))
+        thread = threading.Thread(target=worker_function, name="Sync thread",
+                                  args=syncer_args)
         thread.start()
 
     # Start Tomcat last. It will be the foreground process.

tools/src/main/python/opengrok_tools/reindex_project.py

@@ -94,19 +94,13 @@ def main():
                         help='Set response timeout in seconds for RESTful API calls')
 
     cmd_args = sys.argv[1:]
-    extra_opts = os.environ.get("OPENGROK_INDEXER_OPTIONAL_ARGS")
-    if extra_opts:
-        cmd_args.extend(extra_opts.split())
-
     try:
         args = parser.parse_args(cmd_args)
     except ValueError as e:
         fatal(e)
 
     logger = get_console_logger(get_class_basename(), args.loglevel)
 
-    logger.debug('Command arguments extended with {}'.format(extra_opts))
-
     # Make sure the log directory exists.
     if args.directory:
         if not os.path.isdir(args.directory):
@@ -121,15 +115,19 @@ def main():
                                         args.project)
 
     # Reindex with the modified logging.properties file and read-only config.
-    command = ['-R', conf_file]
-    command.extend(args.options)
+    indexer_options = ['-R', conf_file] + args.options
+    extra_options = os.environ.get("OPENGROK_INDEXER_OPTIONAL_ARGS")
+    if extra_options:
+        logger.debug('indexer arguments extended with {}'.format(extra_options))
+        # Prepend the extra options because we want the arguments to end with a project.
+        indexer_options = extra_options.split() + indexer_options
     java_opts = []
     if args.java_opts:
         java_opts.extend(args.java_opts)
     if logprop_file:
         java_opts.append("-Djava.util.logging.config.file={}".
                          format(logprop_file))
-    indexer = Indexer(command, logger=logger, jar=args.jar,
+    indexer = Indexer(indexer_options, logger=logger, jar=args.jar,
                       java=args.java, java_opts=java_opts,
                       env_vars=args.environment, doprint=args.doprint)
     indexer.execute()