diff --git a/build01/configuration.nix b/build01/configuration.nix
index b16d214..d70acf7 100644
--- a/build01/configuration.nix
+++ b/build01/configuration.nix
@@ -28,6 +28,8 @@ in
../profiles/common.nix
../profiles/docker.nix
+
+ ../services/hound
] ++ userImports;
# /boot is a mirror raid
@@ -40,6 +42,12 @@ in
networking.usePredictableInterfaceNames = false;
networking.dhcpcd.enable = false;
+
+ networking.firewall = {
+ # for Nginx
+ allowedTCPPorts = [ 443 80 ];
+ };
+
systemd.network = {
enable = true;
networks."eth0".extraConfig = ''
@@ -53,6 +61,10 @@ in
'';
};
+ # nginx is being used as the frontend HTTP server for all the services
+ # running on the box
+ services.nginx.enable = true;
+
services.cron.enable = true;
services.cron.systemCronJobs = [
# record that this machine is alive
diff --git a/build01/hydra.nix b/build01/hydra.nix
index 7bcdba9..f5ba53b 100644
--- a/build01/hydra.nix
+++ b/build01/hydra.nix
@@ -66,10 +66,6 @@ in {
};
};
config = {
- networking.firewall = {
- allowedTCPPorts = [ 443 80 ];
- };
-
nixpkgs.config = {
whitelistedLicenses = with lib.licenses; [
unfreeRedistributable
@@ -81,21 +77,18 @@ in {
];
};
- services.nginx = {
- enable = true;
- virtualHosts = {
- "hydra.nix-community.org" = {
- forceSSL = true;
- enableACME = true;
- locations."/" = {
- proxyPass = "http://localhost:${toString(hydraPort)}";
- extraConfig = ''
- proxy_set_header Host $host;
- proxy_set_header X-Real-IP $remote_addr;
- proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
- proxy_set_header X-Forwarded-Proto $scheme;
- '';
- };
+ services.nginx.virtualHosts = {
+ "hydra.nix-community.org" = {
+ forceSSL = true;
+ enableACME = true;
+ locations."/" = {
+ proxyPass = "http://localhost:${toString(hydraPort)}";
+ extraConfig = ''
+ proxy_set_header Host $host;
+ proxy_set_header X-Real-IP $remote_addr;
+ proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
+ proxy_set_header X-Forwarded-Proto $scheme;
+ '';
};
};
};
diff --git a/services/hound/0001-Fail-to-start-if-any-repos-fail-to-index.patch b/services/hound/0001-Fail-to-start-if-any-repos-fail-to-index.patch
new file mode 100644
index 0000000..d3f5f8d
--- /dev/null
+++ b/services/hound/0001-Fail-to-start-if-any-repos-fail-to-index.patch
@@ -0,0 +1,25 @@
+From f976792420843bd079cea3aa85e70cc2cdbe98c2 Mon Sep 17 00:00:00 2001
+From: Graham Christensen <graham@grahamc.com>
+Date: Sun, 8 Oct 2017 10:59:05 -0400
+Subject: [PATCH] Fail to start if any repos fail to index
+
+---
+ cmds/houndd/main.go | 2 +-
+ 1 file changed, 1 insertion(+), 1 deletion(-)
+
+diff --git a/cmds/houndd/main.go b/cmds/houndd/main.go
+index 60152ca..dfba0fc 100644
+--- a/cmds/houndd/main.go
++++ b/cmds/houndd/main.go
+@@ -132,7 +132,7 @@ func main() {
+ log.Panic(err)
+ }
+ if !ok {
+- info_log.Println("Some repos failed to index, see output above")
++ info_log.Fatalf("Some repos failed to index, see output above")
+ } else {
+ info_log.Println("All indexes built!")
+ }
+--
+2.14.2
+
diff --git a/services/hound/0002-Custom-branch-specifier-PR-275.patch b/services/hound/0002-Custom-branch-specifier-PR-275.patch
new file mode 100644
index 0000000..0bdf9c8
--- /dev/null
+++ b/services/hound/0002-Custom-branch-specifier-PR-275.patch
@@ -0,0 +1,222 @@
+From e2fe2b0e4720a74e38258f1bb6bd9ee746bdc6ee Mon Sep 17 00:00:00 2001
+From: Paul Boutes <paul.boutes@gmail.com>
+Date: Fri, 5 Jan 2018 01:52:46 +0100
+Subject: [PATCH 1/3] feat(vcs/git): add ability to use custom branch from vcs
+ config
+
+---
+ vcs/git.go | 29 +++++++++++++++++++++++++----
+ vcs/git_test.go | 38 ++++++++++++++++++++++++++++++++++++++
+ 2 files changed, 63 insertions(+), 4 deletions(-)
+ create mode 100644 vcs/git_test.go
+
+diff --git a/vcs/git.go b/vcs/git.go
+index f8c6682..3626f53 100644
+--- a/vcs/git.go
++++ b/vcs/git.go
+@@ -8,6 +8,7 @@ import (
+ "os/exec"
+ "path/filepath"
+ "strings"
++ "encoding/json"
+ )
+
+ const defaultRef = "master"
+@@ -16,10 +17,29 @@ func init() {
+ Register(newGit, "git")
+ }
+
+-type GitDriver struct{}
++type GitDriver struct {
++ Ref string `json:"ref"`
++}
+
+ func newGit(b []byte) (Driver, error) {
+- return &GitDriver{}, nil
++ d := &GitDriver{}
++ if e := getRef(b, d); e != nil {
++ return nil, e
++ }
++ return d, nil
++}
++
++func getRef(b []byte, d *GitDriver) error {
++ if b != nil {
++ if e := json.Unmarshal(b, d); e != nil {
++ return e
++ }
++ }
++ if d.Ref == "" {
++ d.Ref = defaultRef
++ return nil
++ }
++ return nil
+ }
+
+ func (g *GitDriver) HeadRev(dir string) (string, error) {
+@@ -69,7 +89,7 @@ func (g *GitDriver) Pull(dir string) (string, error) {
+ "--no-tags",
+ "--depth", "1",
+ "origin",
+- fmt.Sprintf("+%s:remotes/origin/%s", defaultRef, defaultRef)); err != nil {
++ fmt.Sprintf("+%s:remotes/origin/%s", g.Ref, g.Ref)); err != nil {
+ return "", err
+ }
+
+@@ -77,7 +97,7 @@ func (g *GitDriver) Pull(dir string) (string, error) {
+ "git",
+ "reset",
+ "--hard",
+- fmt.Sprintf("origin/%s", defaultRef)); err != nil {
++ fmt.Sprintf("origin/%s", g.Ref)); err != nil {
+ return "", err
+ }
+
+@@ -90,6 +110,7 @@ func (g *GitDriver) Clone(dir, url string) (string, error) {
+ "git",
+ "clone",
+ "--depth", "1",
++ "--branch", g.Ref,
+ url,
+ rep)
+ cmd.Dir = par
+diff --git a/vcs/git_test.go b/vcs/git_test.go
+new file mode 100644
+index 0000000..4aa69ad
+--- /dev/null
++++ b/vcs/git_test.go
+@@ -0,0 +1,38 @@
++package vcs
++
++import "testing"
++
++func TestGitConfigWithCustomRef(t *testing.T) {
++ cfg := `{"ref": "custom"}`
++ d, err := New("git", []byte(cfg))
++ if err != nil {
++ t.Fatal(err)
++ }
++ git := d.Driver.(*GitDriver)
++ if git.Ref != "custom" {
++ t.Fatalf("expected branch of \"custom\", got %s", git.Ref)
++ }
++}
++
++func TestGitConfigWithoutRef(t *testing.T) {
++ cfg := `{"option": "option"}`
++ d, err := New("git", []byte(cfg))
++ if err != nil {
++ t.Fatal(err)
++ }
++ git := d.Driver.(*GitDriver)
++ if git.Ref != "master" {
++ t.Fatalf("expected branch of \"master\", got %s", git.Ref)
++ }
++}
++
++func TestGitConfigWithoutAdditionalConfig(t *testing.T) {
++ d, err := New("git", nil)
++ if err != nil {
++ t.Fatal(err)
++ }
++ git := d.Driver.(*GitDriver)
++ if git.Ref != "master" {
++ t.Fatalf("expected branch of \"master\", got %s", git.Ref)
++ }
++}
+
+From efafc6a34ab914e0988398abfa5b0e9eb56f54c5 Mon Sep 17 00:00:00 2001
+From: Paul Boutes <paul.boutes@gmail.com>
+Date: Tue, 9 Jan 2018 00:47:27 +0100
+Subject: [PATCH 2/3] feat: default ref value for GitDriver struct
+
+---
+ vcs/git.go | 18 +++++++-----------
+ 1 file changed, 7 insertions(+), 11 deletions(-)
+
+diff --git a/vcs/git.go b/vcs/git.go
+index 3626f53..5b255e5 100644
+--- a/vcs/git.go
++++ b/vcs/git.go
+@@ -2,13 +2,13 @@ package vcs
+
+ import (
+ "bytes"
++ "encoding/json"
+ "fmt"
+ "io"
+ "log"
+ "os/exec"
+ "path/filepath"
+ "strings"
+- "encoding/json"
+ )
+
+ const defaultRef = "master"
+@@ -22,22 +22,18 @@ type GitDriver struct {
+ }
+
+ func newGit(b []byte) (Driver, error) {
+- d := &GitDriver{}
+- if e := getRef(b, d); e != nil {
++ d := &GitDriver{
++ Ref: defaultRef,
++ }
++ if e := setRefFromConfig(b, d); e != nil {
+ return nil, e
+ }
+ return d, nil
+ }
+
+-func getRef(b []byte, d *GitDriver) error {
++func setRefFromConfig(b []byte, d *GitDriver) error {
+ if b != nil {
+- if e := json.Unmarshal(b, d); e != nil {
+- return e
+- }
+- }
+- if d.Ref == "" {
+- d.Ref = defaultRef
+- return nil
++ return json.Unmarshal(b, d)
+ }
+ return nil
+ }
+
+From 4faf7e7492763208d2337d0102c00f74c950c405 Mon Sep 17 00:00:00 2001
+From: Paul Boutes <paul.boutes@gmail.com>
+Date: Tue, 9 Jan 2018 01:03:08 +0100
+Subject: [PATCH 3/3] feat: remove unecessary function
+
+---
+ vcs/git.go | 14 ++++++--------
+ 1 file changed, 6 insertions(+), 8 deletions(-)
+
+diff --git a/vcs/git.go b/vcs/git.go
+index 5b255e5..364e1d3 100644
+--- a/vcs/git.go
++++ b/vcs/git.go
+@@ -25,17 +25,15 @@ func newGit(b []byte) (Driver, error) {
+ d := &GitDriver{
+ Ref: defaultRef,
+ }
+- if e := setRefFromConfig(b, d); e != nil {
+- return nil, e
++
++ if b == nil {
++ return d, nil
+ }
+- return d, nil
+-}
+
+-func setRefFromConfig(b []byte, d *GitDriver) error {
+- if b != nil {
+- return json.Unmarshal(b, d)
++ if e := json.Unmarshal(b, d); e != nil {
++ return nil, e
+ }
+- return nil
++ return d, nil
+ }
+
+ func (g *GitDriver) HeadRev(dir string) (string, error) {
diff --git a/services/hound/0003-PR-275-p1-Replace-master-in-the-default-base-URL-with-a-rev.patch b/services/hound/0003-PR-275-p1-Replace-master-in-the-default-base-URL-with-a-rev.patch
new file mode 100644
index 0000000..6d2d176
--- /dev/null
+++ b/services/hound/0003-PR-275-p1-Replace-master-in-the-default-base-URL-with-a-rev.patch
@@ -0,0 +1,25 @@
+From 6b4c532f46e88f93d3619f7e3c5983f573510714 Mon Sep 17 00:00:00 2001
+From: Graham Christensen <graham@grahamc.com>
+Date: Tue, 10 Jul 2018 12:55:59 -0400
+Subject: [PATCH] Replace master in the default base URL with a rev
+
+---
+ config/config.go | 2 +-
+ 1 file changed, 1 insertion(+), 1 deletion(-)
+
+diff --git a/config/config.go b/config/config.go
+index ebda60d..b4a29fb 100644
+--- a/config/config.go
++++ b/config/config.go
+@@ -13,7 +13,7 @@ const (
+ defaultPushEnabled = false
+ defaultPollEnabled = true
+ defaultVcs = "git"
+- defaultBaseUrl = "{url}/blob/master/{path}{anchor}"
++ defaultBaseUrl = "{url}/blob/{rev}/{path}{anchor}"
+ defaultAnchor = "#L{line}"
+ )
+
+--
+2.16.4
+
diff --git a/services/hound/README.md b/services/hound/README.md
new file mode 100644
index 0000000..a0b2a78
--- /dev/null
+++ b/services/hound/README.md
@@ -0,0 +1,8 @@
+# search.nix-community.org
+
+This service indexes the code in various repositories in the NixOS
+organization and makes them available for quick querying.
+
+URL: https://search.nix-community.org/
+
+To extend the list of indexed repos, see the ./hound.json file.
diff --git a/services/hound/default.nix b/services/hound/default.nix
new file mode 100644
index 0000000..5fd3461
--- /dev/null
+++ b/services/hound/default.nix
@@ -0,0 +1,29 @@
+{ pkgs, ... }:
+{
+ services.nginx.virtualHosts."search.nix-community.org" = {
+ enableACME = true;
+ forceSSL = true;
+ locations = {
+ "=/open_search.xml".alias = "${./open-search.xml}";
+ "/".proxyPass = "http://127.0.0.1:6080/";
+ };
+ };
+
+ services.hound = {
+ enable = true;
+ listen = "127.0.0.1:6080";
+ config = builtins.readFile ./hound.json;
+ package = pkgs.hound.overrideAttrs (x: {
+ patches = [
+ ./0001-Fail-to-start-if-any-repos-fail-to-index.patch
+ ./0002-Custom-branch-specifier-PR-275.patch
+ ./0003-PR-275-p1-Replace-master-in-the-default-base-URL-with-a-rev.patch
+ ];
+ });
+ };
+
+ systemd.services.hound.serviceConfig = {
+ Restart = "always";
+ RestartSec = 5;
+ };
+}
diff --git a/services/hound/hound.json b/services/hound/hound.json
new file mode 100644
index 0000000..a235220
--- /dev/null
+++ b/services/hound/hound.json
@@ -0,0 +1,228 @@
+{
+ "dbpath": "/var/lib/hound/data",
+ "max-concurrent-indexers": 1,
+ "repos": {
+ "NixOS-.github": {
+ "url": "https://github.com/NixOS/.github.git",
+ "vcs-config": {
+ "ref": "master"
+ }
+ },
+ "NixOS-cabal2nix": {
+ "url": "https://github.com/NixOS/cabal2nix.git",
+ "vcs-config": {
+ "ref": "master"
+ }
+ },
+ "NixOS-flake-registry": {
+ "url": "https://github.com/NixOS/flake-registry.git",
+ "vcs-config": {
+ "ref": "master"
+ }
+ },
+ "NixOS-hydra": {
+ "url": "https://github.com/NixOS/hydra.git",
+ "vcs-config": {
+ "ref": "master"
+ }
+ },
+ "NixOS-hydra-ant-logger": {
+ "url": "https://github.com/NixOS/hydra-ant-logger.git",
+ "vcs-config": {
+ "ref": "master"
+ }
+ },
+ "NixOS-hydra-provisioner": {
+ "url": "https://github.com/NixOS/hydra-provisioner.git",
+ "vcs-config": {
+ "ref": "master"
+ }
+ },
+ "NixOS-mobile-nixos": {
+ "url": "https://github.com/NixOS/mobile-nixos.git",
+ "vcs-config": {
+ "ref": "master"
+ }
+ },
+ "NixOS-mobile-nixos-website": {
+ "url": "https://github.com/NixOS/mobile-nixos-website.git",
+ "vcs-config": {
+ "ref": "master"
+ }
+ },
+ "NixOS-mvn2nix-maven-plugin": {
+ "url": "https://github.com/NixOS/mvn2nix-maven-plugin.git",
+ "vcs-config": {
+ "ref": "master"
+ }
+ },
+ "NixOS-nix": {
+ "url": "https://github.com/NixOS/nix.git",
+ "vcs-config": {
+ "ref": "master"
+ }
+ },
+ "NixOS-nix-eclipse": {
+ "url": "https://github.com/NixOS/nix-eclipse.git",
+ "vcs-config": {
+ "ref": "master"
+ }
+ },
+ "NixOS-nix-idea": {
+ "url": "https://github.com/NixOS/nix-idea.git",
+ "vcs-config": {
+ "ref": "master"
+ }
+ },
+ "NixOS-nix-mode": {
+ "url": "https://github.com/NixOS/nix-mode.git",
+ "vcs-config": {
+ "ref": "master"
+ }
+ },
+ "NixOS-nix-pills": {
+ "url": "https://github.com/NixOS/nix-pills.git",
+ "vcs-config": {
+ "ref": "master"
+ }
+ },
+ "NixOS-nixops": {
+ "url": "https://github.com/NixOS/nixops.git",
+ "vcs-config": {
+ "ref": "master"
+ }
+ },
+ "NixOS-nixops-aws": {
+ "url": "https://github.com/NixOS/nixops-aws.git",
+ "vcs-config": {
+ "ref": "master"
+ }
+ },
+ "NixOS-nixops-hetzner": {
+ "url": "https://github.com/NixOS/nixops-hetzner.git",
+ "vcs-config": {
+ "ref": "master"
+ }
+ },
+ "NixOS-nixos-artwork": {
+ "url": "https://github.com/NixOS/nixos-artwork.git",
+ "vcs-config": {
+ "ref": "master"
+ }
+ },
+ "NixOS-nixos-channel-scripts": {
+ "url": "https://github.com/NixOS/nixos-channel-scripts.git",
+ "vcs-config": {
+ "ref": "master"
+ }
+ },
+ "NixOS-nixos-hardware": {
+ "url": "https://github.com/NixOS/nixos-hardware.git",
+ "vcs-config": {
+ "ref": "master"
+ }
+ },
+ "NixOS-nixos-homepage": {
+ "url": "https://github.com/NixOS/nixos-homepage.git",
+ "vcs-config": {
+ "ref": "master"
+ }
+ },
+ "NixOS-nixos-org-configurations": {
+ "url": "https://github.com/NixOS/nixos-org-configurations.git",
+ "vcs-config": {
+ "ref": "master"
+ }
+ },
+ "NixOS-nixos-planet": {
+ "url": "https://github.com/NixOS/nixos-planet.git",
+ "vcs-config": {
+ "ref": "master"
+ }
+ },
+ "NixOS-nixos-search": {
+ "url": "https://github.com/NixOS/nixos-search.git",
+ "vcs-config": {
+ "ref": "master"
+ }
+ },
+ "NixOS-nixos-weekly": {
+ "url": "https://github.com/NixOS/nixos-weekly.git",
+ "vcs-config": {
+ "ref": "master"
+ }
+ },
+ "NixOS-nixpart": {
+ "url": "https://github.com/NixOS/nixpart.git",
+ "vcs-config": {
+ "ref": "master"
+ }
+ },
+ "NixOS-nixpkgs": {
+ "url": "https://github.com/NixOS/nixpkgs.git",
+ "vcs-config": {
+ "ref": "master"
+ }
+ },
+ "NixOS-npm2nix": {
+ "url": "https://github.com/NixOS/npm2nix.git",
+ "vcs-config": {
+ "ref": "master"
+ }
+ },
+ "NixOS-ofborg": {
+ "url": "https://github.com/NixOS/ofborg.git",
+ "vcs-config": {
+ "ref": "released"
+ }
+ },
+ "NixOS-patchelf": {
+ "url": "https://github.com/NixOS/patchelf.git",
+ "vcs-config": {
+ "ref": "master"
+ }
+ },
+ "NixOS-rfc-steering-committee": {
+ "url": "https://github.com/NixOS/rfc-steering-committee.git",
+ "vcs-config": {
+ "ref": "master"
+ }
+ },
+ "NixOS-rfc39": {
+ "url": "https://github.com/NixOS/rfc39.git",
+ "vcs-config": {
+ "ref": "master"
+ }
+ },
+ "NixOS-rfcs": {
+ "url": "https://github.com/NixOS/rfcs.git",
+ "vcs-config": {
+ "ref": "master"
+ }
+ },
+ "NixOS-security": {
+ "url": "https://github.com/NixOS/security.git",
+ "vcs-config": {
+ "ref": "master"
+ }
+ },
+ "NixOS-snapd-nix-base": {
+ "url": "https://github.com/NixOS/snapd-nix-base.git",
+ "vcs-config": {
+ "ref": "master"
+ }
+ },
+ "NixOS-templates": {
+ "url": "https://github.com/NixOS/templates.git",
+ "vcs-config": {
+ "ref": "master"
+ }
+ },
+ "nixos-users-wiki-wiki": {
+ "url": "https://github.com/nixos-users/wiki.wiki.git",
+ "url-pattern": {
+ "base-url": "{url}/{path}"
+ }
+ }
+ }
+}
diff --git a/services/hound/open-search.xml b/services/hound/open-search.xml
new file mode 100644
index 0000000..7de5ec1
--- /dev/null
+++ b/services/hound/open-search.xml
@@ -0,0 +1,10 @@
+<?xml version="1.0" encoding="UTF-8"?>
+
+<OpenSearchDescription xmlns="http://a9.com/-/spec/opensearch/1.1/">
+ <ShortName>Hound</ShortName>
+ <Description>Search code with Hound</Description>
+ <Tags>Hound</Tags>
+ <Url type="text/html"
+ method="get"
+ template="https://search.nix-community.org/?q={searchTerms}" />
+</OpenSearchDescription>
diff --git a/services/hound/update-hound.py b/services/hound/update-hound.py
new file mode 100755
index 0000000..0bc32d0
--- /dev/null
+++ b/services/hound/update-hound.py
@@ -0,0 +1,61 @@
+#!/usr/bin/env nix-shell
+#!nix-shell -i python3 -p python3Packages.python -p python3Packages.requests
+
+import requests
+import json
+from pprint import pprint
+
+blacklist = [
+ 'https://github.com/NixOS/nixos.git',
+ 'https://github.com/NixOS/systemd.git',
+ 'https://github.com/NixOS/docker.git',
+ 'https://github.com/NixOS/nixpkgs-channels.git',
+ 'https://github.com/NixOS/nixops-dashboard.git',
+ 'https://github.com/NixOS/nixos-foundation.git',
+];
+
+def all_for_org(org, blacklist):
+
+ resp = {}
+
+ next_url = 'https://api.github.com/orgs/{}/repos'.format(org)
+ while next_url is not None:
+ repo_resp = requests.get(next_url)
+
+ if 'next' in repo_resp.links:
+ next_url = repo_resp.links['next']['url']
+ else:
+ next_url = None
+
+ repos = repo_resp.json()
+
+ resp.update({
+ "{}-{}".format(org, repo['name']): {
+ 'url': repo['clone_url'],
+ 'vcs-config': {
+ 'ref': repo['default_branch']
+ }
+ }
+ for repo in repos
+ if repo['clone_url'] not in blacklist
+ })
+
+ return resp
+
+repos = all_for_org('NixOS', blacklist)
+repos['nixos-users-wiki-wiki'] = {
+ "url" : "https://github.com/nixos-users/wiki.wiki.git",
+ "url-pattern" : {
+ "base-url" : "{url}/{path}"
+ }
+}
+
+print(json.dumps(
+ {
+ "max-concurrent-indexers" : 1,
+ "dbpath" : "/var/lib/hound/data",
+ "repos": repos
+ },
+ indent=4,
+ sort_keys=True
+))
diff --git a/terraform/cloudflare_nix-community_org.tf b/terraform/cloudflare_nix-community_org.tf
index a26f3cd..ee6fd0c 100644
--- a/terraform/cloudflare_nix-community_org.tf
+++ b/terraform/cloudflare_nix-community_org.tf
@@ -23,6 +23,13 @@ resource "cloudflare_record" "nix-community-org-hydra-CNAME" {
type = "CNAME"
}
+resource "cloudflare_record" "nix-community-org-search-CNAME" {
+ zone_id = local.nix_community_org_zone_id
+ name = "search"
+ value = "build01.nix-community.com"
+ type = "CNAME"
+}
+
resource "cloudflare_record" "nix-community-org-apex-A" {
zone_id = local.nix_community_org_zone_id
name = "@"