Multiple changes.

 * Sandboxing: enable in open source.
 * Bundled fleetspeak: add commands `fleetspeak_client` and `fleetspeak_server`.
 * Make grr_response_{server,client} depend on fleetspeak-{client,server}-bin packages.
 * `grr_config_updater`: stop using config option `Datastore.implementation`.
 * Implement API layer for recollecting VFS files.
 * Fix client search background color and alignment.
 * Fix CollectMultipleFiles result rendering.

Author: mol123

api_client/python/grr_api_client/connectors/http.py

@@ -453,7 +453,7 @@ def SendStreamingRequest(
     self._CheckResponseStatus(response)
 
     def GenerateChunks() -> Iterator[bytes]:
-      with contextlib.closing(session):
+      with contextlib.closing(session):  # pytype: disable=wrong-arg-types
         with contextlib.closing(response):
           for chunk in response.iter_content(self.DEFAULT_BINARY_CHUNK_SIZE):
             yield chunk

debian/rules

@@ -24,7 +24,8 @@ override_dh_auto_test:
 override_dh_installinit:
 	dh_installinit
 	bash grr/core/scripts/install_server_from_src.sh -i debian/grr-server
-	cp -v -r fleetspeak-server-bin/* debian/grr-server
+	cp -v -r debian/grr-server/usr/share/grr-server/fleetspeak-server-bin/etc/fleetspeak-server debian/grr-server/etc
+	rm -r debian/grr-server/usr/share/grr-server/fleetspeak-server-bin/etc/fleetspeak-server
 
 override_dh_installdocs:
 
@@ -49,6 +50,10 @@ override_dh_installwm:
 override_dh_installxfonts:
 
 override_dh_link:
+	dh_link etc/fleetspeak-server usr/share/grr-server/fleetspeak-server-bin/etc/fleetspeak-server
+	dh_link usr/share/grr-server/fleetspeak-server-bin/usr/bin/fleetspeak-server usr/bin/fleetspeak-server
+	dh_link usr/share/grr-server/fleetspeak-server-bin/usr/bin/fleetspeak-config usr/bin/fleetspeak-config
+	dh_link usr/share/grr-server/fleetspeak-server-bin/lib/systemd/system/fleetspeak-server.service lib/systemd/system/fleetspeak-server.service
 
 override_dh_gconf:
 

grr/client/grr_response_client/distro_entry.py

@@ -19,3 +19,8 @@ def FleetspeakClient():
 def PoolClient():
   from grr_response_client import poolclient
   app.run(poolclient.main)
+
+
+def FleetspeakClientWrapper():
+  from grr_response_client import fleetspeak_client_wrapper
+  app.run(fleetspeak_client_wrapper.main)

grr/client/grr_response_client/fleetspeak_client_wrapper.py

@@ -0,0 +1,120 @@
+#!/usr/bin/env python
+"""Wrapper for running a fleetspeak client in a virtualenv setup.
+
+This script is meant to be used for development.
+
+Requirements for running this script:
+
+ * Running from a virtualenv.
+ * PIP package `fleetspeak-client-bin` is installed.
+ * `grr_config_updater initialize` has been run.
+ * Fleetspeak has been enabled.
+"""
+
+import os
+import platform
+import subprocess
+
+from absl import app
+
+from google.protobuf import text_format
+from grr_response_core import config
+from grr_response_core.lib import config_lib
+from grr_response_core.lib import package
+from grr_response_core.lib import utils
+from grr_response_core.lib.util import temp
+from fleetspeak.src.client.daemonservice.proto.fleetspeak_daemonservice import config_pb2 as fs_daemon_config_pb2
+from fleetspeak.src.client.generic.proto.fleetspeak_client_generic import config_pb2 as fs_cli_config_pb2
+from fleetspeak.src.common.proto.fleetspeak import system_pb2 as fs_system_pb2
+
+
+class Error(Exception):
+  pass
+
+
+def _CreateClientConfig(tmp_dir: str) -> str:
+  """Creates and returns the path to a fleetspeak client config."""
+
+  def TmpPath(*args):
+    return os.path.join(tmp_dir, *args)
+
+  server_config_dir = package.ResourcePath(
+      "fleetspeak-server-bin", "fleetspeak-server-bin/etc/fleetspeak-server")
+  if not os.path.exists(server_config_dir):
+    raise Error(
+        f"Fleetspeak server config dir not found: {server_config_dir}. "
+        "Please make sure `grr_config_updater initialize` has been run.")
+  client_config_name = {
+      "Linux": "linux_client.config",
+      "Windows": "windows_client.config",
+      "Darwin": "darwin_client.config",
+  }
+  client_config_path = os.path.join(server_config_dir,
+                                    client_config_name[platform.system()])
+  with open(client_config_path, "r") as f:
+    client_config = text_format.Parse(f.read(), fs_cli_config_pb2.Config())
+  if client_config.HasField("filesystem_handler"):
+    client_config.filesystem_handler.configuration_directory = TmpPath()
+    # We store the client state file in `Logging.path`.
+    # 1) We need a writable path, where writing a file doesn't surprise the
+    #    user (as opposed to other paths in the source tree).
+    # 2) We need the state file to be somewhat persistent, i.e. kept after
+    #    re-runs of this command. Otherwise the client ID of the client would
+    #    change at each re-run.
+    client_config.filesystem_handler.state_file = os.path.join(
+        config.CONFIG["Logging.path"], "fleetspeak-client.state")
+  with open(TmpPath("config"), "w") as f:
+    f.write(text_format.MessageToString(client_config))
+  return TmpPath("config")
+
+
+def _CreateServiceConfig(config_dir: str) -> None:
+  """Creates a fleetspeak service config in the config directory."""
+  service_config_path = config.CONFIG["ClientBuilder.fleetspeak_config_path"]
+  with open(service_config_path, "r") as f:
+    data = config.CONFIG.InterpolateValue(f.read())
+    service_config = text_format.Parse(data,
+                                       fs_system_pb2.ClientServiceConfig())
+  daemon_config = fs_daemon_config_pb2.Config()
+  service_config.config.Unpack(daemon_config)
+  del daemon_config.argv[:]
+  daemon_config.argv.extend([
+      "grr_client",
+  ])
+  service_config.config.Pack(daemon_config)
+  utils.EnsureDirExists(os.path.join(config_dir, "textservices"))
+  with open(os.path.join(config_dir, "textservices", "GRR.textproto"),
+            "w") as f:
+    f.write(text_format.MessageToString(service_config))
+
+
+def _RunClient(tmp_dir: str) -> None:
+  """Runs the fleetspeak client."""
+  config_path = _CreateClientConfig(tmp_dir)
+  _CreateServiceConfig(tmp_dir)
+  fleetspeak_client = package.ResourcePath(
+      "fleetspeak-client-bin",
+      "fleetspeak-client-bin/usr/bin/fleetspeak-client")
+  if not fleetspeak_client or not os.path.exists(fleetspeak_client):
+    raise Error(
+        f"Fleetspeak client binary not found: {fleetspeak_client}."
+        "Please make sure that the package `fleetspeak-client-bin` has been "
+        "installed.")
+  command = [
+      fleetspeak_client,
+      "--logtostderr",
+      "-config",
+      config_path,
+  ]
+  subprocess.check_call(command)
+
+
+def main(argv):
+  del argv  # unused
+  config_lib.ParseConfigCommandLine()
+  with temp.AutoTempDirPath(remove_non_empty=True) as tmp_dir:
+    _RunClient(tmp_dir)
+
+
+if __name__ == "__main__":
+  app.run(main)

grr/client/setup.py

@@ -119,7 +119,9 @@ def run(self):
             "grr_client = grr_response_client.distro_entry:Client",
             ("grr_fleetspeak_client = "
              "grr_response_client.distro_entry:FleetspeakClient"),
-            "grr_pool_client = grr_response_client.distro_entry:PoolClient"
+            "grr_pool_client = grr_response_client.distro_entry:PoolClient",
+            ("fleetspeak_client = "
+             "grr_response_client.distro_entry:FleetspeakClientWrapper"),
         ]
     },
     cmdclass={
@@ -136,6 +138,7 @@ def run(self):
         PYTSK3,
         "retry==0.9.2",
         "libfsntfs-python==20210503",
+        "fleetspeak-client-bin==0.1.9",
     ],
     extras_require={
         # The following requirements are needed in Windows.

grr/core/grr_response_core/lib/util/sqlite.py

@@ -37,7 +37,7 @@ def Query(self, query: Text) -> Iterator[Tuple]:  # pylint: disable=g-bare-gener
     Yields:
       Database rows that are results of the query.
     """
-    with contextlib.closing(self._conn.cursor()) as cursor:
+    with contextlib.closing(self._conn.cursor()) as cursor:  # pytype: disable=wrong-arg-types
       cursor.execute(query)
 
       while True:
@@ -65,7 +65,7 @@ def IOConnection(db_filedesc: IO[bytes]) -> Iterator[ConnectionContext]:
     with io.open(temp_db_filepath, mode="wb") as temp_db_filedesc:
       _CopyIO(input=db_filedesc, output=temp_db_filedesc)
 
-    with contextlib.closing(sqlite3.connect(temp_db_filepath)) as conn:
+    with contextlib.closing(sqlite3.connect(temp_db_filepath)) as conn:  # pytype: disable=wrong-arg-types
       yield ConnectionContext(conn)
 
 

grr/core/install_data/etc/grr-server.yaml

@@ -196,6 +196,11 @@ Platform:Darwin:
 
   Client.platform: darwin
 
+  Global Install Context:
+    Client.unprivileged_user: nobody
+    Client.unprivileged_group: nobody
+    Client.use_filesystem_sandboxing: True
+
   Client Context:
     Logging.engines: stderr,file,syslog
 
@@ -206,6 +211,12 @@ Platform:Linux:
 
   Client.platform: linux
 
+  Global Install Context:
+    Client.unprivileged_user: _%(Client.name)
+    Client.unprivileged_group: _%(Client.name)
+    Client.use_filesystem_sandboxing: True
+    Client.use_memory_sandboxing: True
+
   Client Context:
 
     Logging.engines: stderr,file,syslog
@@ -233,6 +244,10 @@ Platform:Windows:
     - "%(install_path)"
   Client.grr_tempdir: "Temp"
 
+  Global Install Context:
+    Client.use_filesystem_sandboxing: True
+    Client.use_memory_sandboxing: True
+
   Arch:amd64:
     ClientBuilder.vs_arch: x64
 

grr/server/grr_response_server/bin/config_updater_util.py

@@ -5,7 +5,6 @@
 import getpass
 import os
 import re
-import shutil
 import socket
 import subprocess
 import sys
@@ -27,6 +26,7 @@
 from grr_api_client import root as api_root
 from grr_response_client_builder import repacking
 from grr_response_core import config as grr_config
+from grr_response_core.lib import package
 from grr_response_core.lib.util import compatibility
 from grr_response_server import maintenance_utils
 from grr_response_server import server_startup
@@ -331,7 +331,11 @@ def __init__(self):
     self.mysql_port = 3306
     self.mysql_database: str = None
     self.mysql_unix_socket: str = None
-    self.config_dir = "/etc/fleetspeak-server"
+    self.config_dir = package.ResourcePath(
+        "fleetspeak-server-bin", "fleetspeak-server-bin/etc/fleetspeak-server")
+    self._fleetspeak_config_command_path = package.ResourcePath(
+        "fleetspeak-server-bin",
+        "fleetspeak-server-bin/usr/bin/fleetspeak-config")
 
   def Prompt(self, config):
     """Sets up the in-memory configuration interactively."""
@@ -372,9 +376,10 @@ def _ConfigPath(self, *path_components: str) -> str:
     return os.path.join(self.config_dir, *path_components)
 
   def _IsFleetspeakPresent(self) -> bool:
+    """Returns True, if a fleetspeak server is available on this system."""
     if not os.path.exists(self._ConfigPath()):
       return False
-    if not shutil.which("fleetspeak-config"):
+    if not os.path.exists(self._fleetspeak_config_command_path):
       return False
     return True
 
@@ -477,8 +482,9 @@ def _WriteEnabled(self, config):
     cp.darwin_client_configuration_file = self._ConfigPath(
         "darwin_client.config")
 
-    p = subprocess.Popen(["fleetspeak-config", "-config", "/dev/stdin"],
-                         stdin=subprocess.PIPE)
+    p = subprocess.Popen(
+        [self._fleetspeak_config_command_path, "-config", "/dev/stdin"],
+        stdin=subprocess.PIPE)
     p.communicate(input=text_format.MessageToString(cp).encode())
     if p.wait() != 0:
       raise RuntimeError("fleetspeak-config command failed.")
@@ -490,10 +496,10 @@ def _WriteEnabled(self, config):
     # pylint: enable=g-import-not-at-top
 
     if (os.geteuid() == 0 and pwd.getpwnam("fleetspeak") and
-        grp.getgrnam("fleetspeak")):
+        grp.getgrnam("fleetspeak") and
+        os.path.exists("/etc/fleetspeak-server")):
       subprocess.check_call(
-          ["chown", "-R", "fleetspeak:fleetspeak",
-           self._ConfigPath()])
+          ["chown", "-R", "fleetspeak:fleetspeak", "/etc/fleetspeak-server"])
 
     try:
       os.unlink(self._ConfigPath("disabled"))
@@ -577,35 +583,7 @@ def ConfigureDatastore(config):
         "For GRR to work each GRR server has to be able to communicate with\n"
         "the datastore. To do this we need to configure a datastore.\n")
 
-  existing_datastore = grr_config.CONFIG.Get("Datastore.implementation")
-
-  if not existing_datastore or existing_datastore == "FakeDataStore":
-    ConfigureMySQLDatastore(config)
-    return
-
-  print("Found existing settings:\n  REL_DB MySQL database")
-  if existing_datastore == "SqliteDataStore":
-    set_up_mysql = RetryBoolQuestion(
-        "The SQLite datastore is no longer supported. Would you like to\n"
-        "set up a MySQL datastore? Answering 'no' will abort config "
-        "initialization.", True)
-    if set_up_mysql:
-      print("\nPlease note that no data will be migrated from SQLite to "
-            "MySQL.\n")
-      ConfigureMySQLDatastore(config)
-    else:
-      raise ConfigInitError()
-  elif existing_datastore == "MySQLAdvancedDataStore":
-    set_up_mysql = RetryBoolQuestion(
-        "The MySQLAdvancedDataStore is no longer supported. Would you like to\n"
-        "set up a new MySQL datastore? Answering 'no' will abort config "
-        "initialization.", True)
-    if set_up_mysql:
-      print("\nPlease note that no data will be migrated from the old data "
-            "store.\n")
-      ConfigureMySQLDatastore(config)
-    else:
-      raise ConfigInitError()
+  ConfigureMySQLDatastore(config)
 
 
 def ConfigureUrls(config, external_hostname: Optional[Text] = None):

grr/server/grr_response_server/bin/fleetspeak_server_wrapper.py

@@ -0,0 +1,54 @@
+#!/usr/bin/env python
+"""Wrapper for running a fleetspeak server in a virtualenv setup.
+
+This script is meant to be used for development.
+
+Requirements for running this script:
+
+ * Running from a virtualenv.
+ * PIP package `fletspeak-server-bin` is installed.
+ * `grr_config_updater initialize` has been run.
+ * Fleetspeak has been enabled.
+"""
+import os
+import subprocess
+
+from absl import app
+
+from grr_response_core.lib import config_lib
+from grr_response_core.lib import package
+
+
+class Error(Exception):
+  pass
+
+
+def main(argv):
+  config_lib.ParseConfigCommandLine()
+  config_dir = package.ResourcePath(
+      "fleetspeak-server-bin", "fleetspeak-server-bin/etc/fleetspeak-server")
+  if not os.path.exists(config_dir):
+    raise Error(
+        f"Configuration directory not found: {config_dir}. "
+        "Please make sure `grr_config_updater initialize` has been run.")
+  fleetspeak_server = package.ResourcePath(
+      "fleetspeak-server-bin",
+      "fleetspeak-server-bin/usr/bin/fleetspeak-server")
+  if not os.path.exists(fleetspeak_server):
+    raise Error(
+        f"Fleetspeak server binary not found: {fleetspeak_server}. "
+        "Please make sure that the package `fleetspeak-server-bin` has been "
+        "installed.")
+  command = [
+      fleetspeak_server,
+      "--logtostderr",
+      "--services_config",
+      os.path.join(config_dir, "server.services.config"),
+      "--components_config",
+      os.path.join(config_dir, "server.components.config"),
+  ]
+  subprocess.check_call(command)
+
+
+if __name__ == "__main__":
+  app.run(main)