deploy search.nix-community.org

This is migrating search.nix.gcs.io with the code from
https://github.com/grahamc/network/tree/master/flexo/hound

build01/configuration.nix

@@ -28,6 +28,8 @@ in
 
     ../profiles/common.nix
     ../profiles/docker.nix
+
+    ../services/hound
   ] ++ userImports;
 
   # /boot is a mirror raid
@@ -40,6 +42,12 @@ in
 
   networking.usePredictableInterfaceNames = false;
   networking.dhcpcd.enable = false;
+
+  networking.firewall = {
+    # for Nginx
+    allowedTCPPorts = [ 443 80 ];
+  };
+
   systemd.network = {
     enable = true;
     networks."eth0".extraConfig = ''
@@ -53,6 +61,10 @@ in
     '';
   };
 
+  # nginx is being used as the frontend HTTP server for all the services
+  # running on the box
+  services.nginx.enable = true;
+
   services.cron.enable = true;
   services.cron.systemCronJobs = [
     # record that this machine is alive

build01/hydra.nix

@@ -66,10 +66,6 @@ in {
     };
   };
   config = {
-    networking.firewall = {
-      allowedTCPPorts = [ 443 80 ];
-    };
-
     nixpkgs.config = {
       whitelistedLicenses = with lib.licenses; [
         unfreeRedistributable
@@ -81,21 +77,18 @@ in {
       ];
     };
 
-    services.nginx = {
+    services.nginx.virtualHosts = {
-      enable = true;
+      "hydra.nix-community.org" = {
-      virtualHosts = {
+        forceSSL = true;
-        "hydra.nix-community.org" = {
+        enableACME = true;
-          forceSSL = true;
+        locations."/" = {
-          enableACME = true;
+          proxyPass = "http://localhost:${toString(hydraPort)}";
-          locations."/" = {
+          extraConfig = ''
-            proxyPass = "http://localhost:${toString(hydraPort)}";
+            proxy_set_header Host $host;
-            extraConfig = ''
+            proxy_set_header X-Real-IP $remote_addr;
-              proxy_set_header Host $host;
+            proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
-              proxy_set_header X-Real-IP $remote_addr;
+            proxy_set_header X-Forwarded-Proto $scheme;
-              proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
+          '';
-              proxy_set_header X-Forwarded-Proto $scheme;
-            '';
-          };
         };
       };
     };

services/hound/0001-Fail-to-start-if-any-repos-fail-to-index.patch

@@ -0,0 +1,25 @@
+From f976792420843bd079cea3aa85e70cc2cdbe98c2 Mon Sep 17 00:00:00 2001
+From: Graham Christensen <graham@grahamc.com>
+Date: Sun, 8 Oct 2017 10:59:05 -0400
+Subject: [PATCH] Fail to start if any repos fail to index
+
+---
+ cmds/houndd/main.go | 2 +-
+ 1 file changed, 1 insertion(+), 1 deletion(-)
+
+diff --git a/cmds/houndd/main.go b/cmds/houndd/main.go
+index 60152ca..dfba0fc 100644
+--- a/cmds/houndd/main.go
++++ b/cmds/houndd/main.go
+@@ -132,7 +132,7 @@ func main() {
+ 		log.Panic(err)
+ 	}
+ 	if !ok {
+-		info_log.Println("Some repos failed to index, see output above")
++		info_log.Fatalf("Some repos failed to index, see output above")
+ 	} else {
+ 		info_log.Println("All indexes built!")
+ 	}
+-- 
+2.14.2
+

services/hound/0002-Custom-branch-specifier-PR-275.patch

@@ -0,0 +1,222 @@
+From e2fe2b0e4720a74e38258f1bb6bd9ee746bdc6ee Mon Sep 17 00:00:00 2001
+From: Paul Boutes <paul.boutes@gmail.com>
+Date: Fri, 5 Jan 2018 01:52:46 +0100
+Subject: [PATCH 1/3] feat(vcs/git): add ability to use custom branch from vcs
+ config
+
+---
+ vcs/git.go      | 29 +++++++++++++++++++++++++----
+ vcs/git_test.go | 38 ++++++++++++++++++++++++++++++++++++++
+ 2 files changed, 63 insertions(+), 4 deletions(-)
+ create mode 100644 vcs/git_test.go
+
+diff --git a/vcs/git.go b/vcs/git.go
+index f8c6682..3626f53 100644
+--- a/vcs/git.go
++++ b/vcs/git.go
+@@ -8,6 +8,7 @@ import (
+ 	"os/exec"
+ 	"path/filepath"
+ 	"strings"
++	"encoding/json"
+ )
+ 
+ const defaultRef = "master"
+@@ -16,10 +17,29 @@ func init() {
+ 	Register(newGit, "git")
+ }
+ 
+-type GitDriver struct{}
++type GitDriver struct {
++	Ref string `json:"ref"`
++}
+ 
+ func newGit(b []byte) (Driver, error) {
+-	return &GitDriver{}, nil
++	d := &GitDriver{}
++	if e := getRef(b, d); e != nil {
++		return nil, e
++	}
++	return d, nil
++}
++
++func getRef(b []byte, d *GitDriver) error {
++	if b != nil {
++		if e := json.Unmarshal(b, d); e != nil {
++			return e
++		}
++	}
++	if d.Ref == "" {
++		d.Ref = defaultRef
++		return nil
++	}
++	return nil
+ }
+ 
+ func (g *GitDriver) HeadRev(dir string) (string, error) {
+@@ -69,7 +89,7 @@ func (g *GitDriver) Pull(dir string) (string, error) {
+ 		"--no-tags",
+ 		"--depth", "1",
+ 		"origin",
+-		fmt.Sprintf("+%s:remotes/origin/%s", defaultRef, defaultRef)); err != nil {
++		fmt.Sprintf("+%s:remotes/origin/%s", g.Ref, g.Ref)); err != nil {
+ 		return "", err
+ 	}
+ 
+@@ -77,7 +97,7 @@ func (g *GitDriver) Pull(dir string) (string, error) {
+ 		"git",
+ 		"reset",
+ 		"--hard",
+-		fmt.Sprintf("origin/%s", defaultRef)); err != nil {
++		fmt.Sprintf("origin/%s", g.Ref)); err != nil {
+ 		return "", err
+ 	}
+ 
+@@ -90,6 +110,7 @@ func (g *GitDriver) Clone(dir, url string) (string, error) {
+ 		"git",
+ 		"clone",
+ 		"--depth", "1",
++		"--branch", g.Ref,
+ 		url,
+ 		rep)
+ 	cmd.Dir = par
+diff --git a/vcs/git_test.go b/vcs/git_test.go
+new file mode 100644
+index 0000000..4aa69ad
+--- /dev/null
++++ b/vcs/git_test.go
+@@ -0,0 +1,38 @@
++package vcs
++
++import "testing"
++
++func TestGitConfigWithCustomRef(t *testing.T) {
++	cfg := `{"ref": "custom"}`
++	d, err := New("git", []byte(cfg))
++	if err != nil {
++		t.Fatal(err)
++	}
++	git := d.Driver.(*GitDriver)
++	if git.Ref != "custom" {
++		t.Fatalf("expected branch of \"custom\", got %s", git.Ref)
++	}
++}
++
++func TestGitConfigWithoutRef(t *testing.T) {
++	cfg := `{"option": "option"}`
++	d, err := New("git", []byte(cfg))
++	if err != nil {
++		t.Fatal(err)
++	}
++	git := d.Driver.(*GitDriver)
++	if git.Ref != "master" {
++		t.Fatalf("expected branch of \"master\", got %s", git.Ref)
++	}
++}
++
++func TestGitConfigWithoutAdditionalConfig(t *testing.T) {
++	d, err := New("git", nil)
++	if err != nil {
++		t.Fatal(err)
++	}
++	git := d.Driver.(*GitDriver)
++	if git.Ref != "master" {
++		t.Fatalf("expected branch of \"master\", got %s", git.Ref)
++	}
++}
+
+From efafc6a34ab914e0988398abfa5b0e9eb56f54c5 Mon Sep 17 00:00:00 2001
+From: Paul Boutes <paul.boutes@gmail.com>
+Date: Tue, 9 Jan 2018 00:47:27 +0100
+Subject: [PATCH 2/3] feat: default ref value for GitDriver struct
+
+---
+ vcs/git.go | 18 +++++++-----------
+ 1 file changed, 7 insertions(+), 11 deletions(-)
+
+diff --git a/vcs/git.go b/vcs/git.go
+index 3626f53..5b255e5 100644
+--- a/vcs/git.go
++++ b/vcs/git.go
+@@ -2,13 +2,13 @@ package vcs
+ 
+ import (
+ 	"bytes"
++	"encoding/json"
+ 	"fmt"
+ 	"io"
+ 	"log"
+ 	"os/exec"
+ 	"path/filepath"
+ 	"strings"
+-	"encoding/json"
+ )
+ 
+ const defaultRef = "master"
+@@ -22,22 +22,18 @@ type GitDriver struct {
+ }
+ 
+ func newGit(b []byte) (Driver, error) {
+-	d := &GitDriver{}
+-	if e := getRef(b, d); e != nil {
++	d := &GitDriver{
++		Ref: defaultRef,
++	}
++	if e := setRefFromConfig(b, d); e != nil {
+ 		return nil, e
+ 	}
+ 	return d, nil
+ }
+ 
+-func getRef(b []byte, d *GitDriver) error {
++func setRefFromConfig(b []byte, d *GitDriver) error {
+ 	if b != nil {
+-		if e := json.Unmarshal(b, d); e != nil {
+-			return e
+-		}
+-	}
+-	if d.Ref == "" {
+-		d.Ref = defaultRef
+-		return nil
++		return json.Unmarshal(b, d)
+ 	}
+ 	return nil
+ }
+
+From 4faf7e7492763208d2337d0102c00f74c950c405 Mon Sep 17 00:00:00 2001
+From: Paul Boutes <paul.boutes@gmail.com>
+Date: Tue, 9 Jan 2018 01:03:08 +0100
+Subject: [PATCH 3/3] feat: remove unecessary function
+
+---
+ vcs/git.go | 14 ++++++--------
+ 1 file changed, 6 insertions(+), 8 deletions(-)
+
+diff --git a/vcs/git.go b/vcs/git.go
+index 5b255e5..364e1d3 100644
+--- a/vcs/git.go
++++ b/vcs/git.go
+@@ -25,17 +25,15 @@ func newGit(b []byte) (Driver, error) {
+ 	d := &GitDriver{
+ 		Ref: defaultRef,
+ 	}
+-	if e := setRefFromConfig(b, d); e != nil {
+-		return nil, e
++
++	if b == nil {
++		return d, nil
+ 	}
+-	return d, nil
+-}
+ 
+-func setRefFromConfig(b []byte, d *GitDriver) error {
+-	if b != nil {
+-		return json.Unmarshal(b, d)
++	if e := json.Unmarshal(b, d); e != nil {
++		return nil, e
+ 	}
+-	return nil
++	return d, nil
+ }
+ 
+ func (g *GitDriver) HeadRev(dir string) (string, error) {

services/hound/0003-PR-275-p1-Replace-master-in-the-default-base-URL-with-a-rev.patch

@@ -0,0 +1,25 @@
+From 6b4c532f46e88f93d3619f7e3c5983f573510714 Mon Sep 17 00:00:00 2001
+From: Graham Christensen <graham@grahamc.com>
+Date: Tue, 10 Jul 2018 12:55:59 -0400
+Subject: [PATCH] Replace master in the default base URL with a rev
+
+---
+ config/config.go | 2 +-
+ 1 file changed, 1 insertion(+), 1 deletion(-)
+
+diff --git a/config/config.go b/config/config.go
+index ebda60d..b4a29fb 100644
+--- a/config/config.go
++++ b/config/config.go
+@@ -13,7 +13,7 @@ const (
+ 	defaultPushEnabled           = false
+ 	defaultPollEnabled           = true
+ 	defaultVcs                   = "git"
+-	defaultBaseUrl               = "{url}/blob/master/{path}{anchor}"
++	defaultBaseUrl               = "{url}/blob/{rev}/{path}{anchor}"
+ 	defaultAnchor                = "#L{line}"
+ )
+ 
+-- 
+2.16.4
+

services/hound/README.md

@@ -0,0 +1,8 @@
+# search.nix-community.org
+
+This service indexes the code in various repositories in the NixOS
+organization and makes them available for quick querying.
+
+URL: https://search.nix-community.org/
+
+To extend the list of indexed repos, see the ./hound.json file.

services/hound/default.nix

@@ -0,0 +1,29 @@
+{ pkgs, ... }:
+{
+  services.nginx.virtualHosts."search.nix-community.org" = {
+    enableACME = true;
+    forceSSL = true;
+    locations = {
+      "=/open_search.xml".alias = "${./open-search.xml}";
+      "/".proxyPass = "http://127.0.0.1:6080/";
+    };
+  };
+
+  services.hound = {
+    enable = true;
+    listen = "127.0.0.1:6080";
+    config = builtins.readFile ./hound.json;
+    package = pkgs.hound.overrideAttrs (x: {
+      patches = [
+        ./0001-Fail-to-start-if-any-repos-fail-to-index.patch
+        ./0002-Custom-branch-specifier-PR-275.patch
+        ./0003-PR-275-p1-Replace-master-in-the-default-base-URL-with-a-rev.patch
+      ];
+    });
+  };
+
+  systemd.services.hound.serviceConfig = {
+    Restart = "always";
+    RestartSec = 5;
+  };
+}

services/hound/hound.json

@@ -0,0 +1,228 @@
+{
+    "dbpath": "/var/lib/hound/data",
+    "max-concurrent-indexers": 1,
+    "repos": {
+        "NixOS-.github": {
+            "url": "https://github.com/NixOS/.github.git",
+            "vcs-config": {
+                "ref": "master"
+            }
+        },
+        "NixOS-cabal2nix": {
+            "url": "https://github.com/NixOS/cabal2nix.git",
+            "vcs-config": {
+                "ref": "master"
+            }
+        },
+        "NixOS-flake-registry": {
+            "url": "https://github.com/NixOS/flake-registry.git",
+            "vcs-config": {
+                "ref": "master"
+            }
+        },
+        "NixOS-hydra": {
+            "url": "https://github.com/NixOS/hydra.git",
+            "vcs-config": {
+                "ref": "master"
+            }
+        },
+        "NixOS-hydra-ant-logger": {
+            "url": "https://github.com/NixOS/hydra-ant-logger.git",
+            "vcs-config": {
+                "ref": "master"
+            }
+        },
+        "NixOS-hydra-provisioner": {
+            "url": "https://github.com/NixOS/hydra-provisioner.git",
+            "vcs-config": {
+                "ref": "master"
+            }
+        },
+        "NixOS-mobile-nixos": {
+            "url": "https://github.com/NixOS/mobile-nixos.git",
+            "vcs-config": {
+                "ref": "master"
+            }
+        },
+        "NixOS-mobile-nixos-website": {
+            "url": "https://github.com/NixOS/mobile-nixos-website.git",
+            "vcs-config": {
+                "ref": "master"
+            }
+        },
+        "NixOS-mvn2nix-maven-plugin": {
+            "url": "https://github.com/NixOS/mvn2nix-maven-plugin.git",
+            "vcs-config": {
+                "ref": "master"
+            }
+        },
+        "NixOS-nix": {
+            "url": "https://github.com/NixOS/nix.git",
+            "vcs-config": {
+                "ref": "master"
+            }
+        },
+        "NixOS-nix-eclipse": {
+            "url": "https://github.com/NixOS/nix-eclipse.git",
+            "vcs-config": {
+                "ref": "master"
+            }
+        },
+        "NixOS-nix-idea": {
+            "url": "https://github.com/NixOS/nix-idea.git",
+            "vcs-config": {
+                "ref": "master"
+            }
+        },
+        "NixOS-nix-mode": {
+            "url": "https://github.com/NixOS/nix-mode.git",
+            "vcs-config": {
+                "ref": "master"
+            }
+        },
+        "NixOS-nix-pills": {
+            "url": "https://github.com/NixOS/nix-pills.git",
+            "vcs-config": {
+                "ref": "master"
+            }
+        },
+        "NixOS-nixops": {
+            "url": "https://github.com/NixOS/nixops.git",
+            "vcs-config": {
+                "ref": "master"
+            }
+        },
+        "NixOS-nixops-aws": {
+            "url": "https://github.com/NixOS/nixops-aws.git",
+            "vcs-config": {
+                "ref": "master"
+            }
+        },
+        "NixOS-nixops-hetzner": {
+            "url": "https://github.com/NixOS/nixops-hetzner.git",
+            "vcs-config": {
+                "ref": "master"
+            }
+        },
+        "NixOS-nixos-artwork": {
+            "url": "https://github.com/NixOS/nixos-artwork.git",
+            "vcs-config": {
+                "ref": "master"
+            }
+        },
+        "NixOS-nixos-channel-scripts": {
+            "url": "https://github.com/NixOS/nixos-channel-scripts.git",
+            "vcs-config": {
+                "ref": "master"
+            }
+        },
+        "NixOS-nixos-hardware": {
+            "url": "https://github.com/NixOS/nixos-hardware.git",
+            "vcs-config": {
+                "ref": "master"
+            }
+        },
+        "NixOS-nixos-homepage": {
+            "url": "https://github.com/NixOS/nixos-homepage.git",
+            "vcs-config": {
+                "ref": "master"
+            }
+        },
+        "NixOS-nixos-org-configurations": {
+            "url": "https://github.com/NixOS/nixos-org-configurations.git",
+            "vcs-config": {
+                "ref": "master"
+            }
+        },
+        "NixOS-nixos-planet": {
+            "url": "https://github.com/NixOS/nixos-planet.git",
+            "vcs-config": {
+                "ref": "master"
+            }
+        },
+        "NixOS-nixos-search": {
+            "url": "https://github.com/NixOS/nixos-search.git",
+            "vcs-config": {
+                "ref": "master"
+            }
+        },
+        "NixOS-nixos-weekly": {
+            "url": "https://github.com/NixOS/nixos-weekly.git",
+            "vcs-config": {
+                "ref": "master"
+            }
+        },
+        "NixOS-nixpart": {
+            "url": "https://github.com/NixOS/nixpart.git",
+            "vcs-config": {
+                "ref": "master"
+            }
+        },
+        "NixOS-nixpkgs": {
+            "url": "https://github.com/NixOS/nixpkgs.git",
+            "vcs-config": {
+                "ref": "master"
+            }
+        },
+        "NixOS-npm2nix": {
+            "url": "https://github.com/NixOS/npm2nix.git",
+            "vcs-config": {
+                "ref": "master"
+            }
+        },
+        "NixOS-ofborg": {
+            "url": "https://github.com/NixOS/ofborg.git",
+            "vcs-config": {
+                "ref": "released"
+            }
+        },
+        "NixOS-patchelf": {
+            "url": "https://github.com/NixOS/patchelf.git",
+            "vcs-config": {
+                "ref": "master"
+            }
+        },
+        "NixOS-rfc-steering-committee": {
+            "url": "https://github.com/NixOS/rfc-steering-committee.git",
+            "vcs-config": {
+                "ref": "master"
+            }
+        },
+        "NixOS-rfc39": {
+            "url": "https://github.com/NixOS/rfc39.git",
+            "vcs-config": {
+                "ref": "master"
+            }
+        },
+        "NixOS-rfcs": {
+            "url": "https://github.com/NixOS/rfcs.git",
+            "vcs-config": {
+                "ref": "master"
+            }
+        },
+        "NixOS-security": {
+            "url": "https://github.com/NixOS/security.git",
+            "vcs-config": {
+                "ref": "master"
+            }
+        },
+        "NixOS-snapd-nix-base": {
+            "url": "https://github.com/NixOS/snapd-nix-base.git",
+            "vcs-config": {
+                "ref": "master"
+            }
+        },
+        "NixOS-templates": {
+            "url": "https://github.com/NixOS/templates.git",
+            "vcs-config": {
+                "ref": "master"
+            }
+        },
+        "nixos-users-wiki-wiki": {
+            "url": "https://github.com/nixos-users/wiki.wiki.git",
+            "url-pattern": {
+                "base-url": "{url}/{path}"
+            }
+        }
+    }
+}

services/hound/open-search.xml

@@ -0,0 +1,10 @@
+<?xml version="1.0" encoding="UTF-8"?>
+
+<OpenSearchDescription xmlns="http://a9.com/-/spec/opensearch/1.1/">
+    <ShortName>Hound</ShortName>
+    <Description>Search code with Hound</Description>
+    <Tags>Hound</Tags>
+    <Url type="text/html"
+         method="get"
+         template="https://search.nix-community.org/?q={searchTerms}" />
+</OpenSearchDescription>

services/hound/update-hound.py

@@ -0,0 +1,61 @@
+#!/usr/bin/env nix-shell
+#!nix-shell -i python3 -p python3Packages.python -p python3Packages.requests
+
+import requests
+import json
+from pprint import pprint
+
+blacklist = [
+    'https://github.com/NixOS/nixos.git',
+    'https://github.com/NixOS/systemd.git',
+    'https://github.com/NixOS/docker.git',
+    'https://github.com/NixOS/nixpkgs-channels.git',
+    'https://github.com/NixOS/nixops-dashboard.git',
+    'https://github.com/NixOS/nixos-foundation.git',
+];
+
+def all_for_org(org, blacklist):
+
+    resp = {}
+
+    next_url = 'https://api.github.com/orgs/{}/repos'.format(org)
+    while next_url is not None:
+        repo_resp = requests.get(next_url)
+
+        if 'next' in repo_resp.links:
+            next_url = repo_resp.links['next']['url']
+        else:
+            next_url = None
+
+        repos = repo_resp.json()
+
+        resp.update({
+            "{}-{}".format(org, repo['name']): {
+                'url': repo['clone_url'],
+                'vcs-config': {
+                    'ref': repo['default_branch']
+                }
+            }
+            for repo in repos
+            if repo['clone_url'] not in blacklist
+        })
+
+    return resp
+
+repos = all_for_org('NixOS', blacklist)
+repos['nixos-users-wiki-wiki'] = {
+    "url" : "https://github.com/nixos-users/wiki.wiki.git",
+    "url-pattern" : {
+        "base-url" : "{url}/{path}"
+    }
+}
+
+print(json.dumps(
+    {
+        "max-concurrent-indexers" : 1,
+        "dbpath" : "/var/lib/hound/data",
+        "repos": repos
+    },
+    indent=4,
+    sort_keys=True
+))

terraform/cloudflare_nix-community_org.tf

@@ -23,6 +23,13 @@ resource "cloudflare_record" "nix-community-org-hydra-CNAME" {
   type    = "CNAME"
 }
 
+resource "cloudflare_record" "nix-community-org-search-CNAME" {
+  zone_id = local.nix_community_org_zone_id
+  name    = "search"
+  value   = "build01.nix-community.com"
+  type    = "CNAME"
+}
+
 resource "cloudflare_record" "nix-community-org-apex-A" {
   zone_id = local.nix_community_org_zone_id
   name    = "@"