merged
authorwenzelm
Tue Nov 05 22:56:06 2019 +0100 (8 days ago ago)
changeset 712579b531e611d66
parent 71243 9858f391ed2d
parent 71256 6ca9e8377613
child 71258 1d19e844fa4d
child 71259 295609359b58
merged
     1.1 --- a/Admin/Phabricator/README	Tue Nov 05 21:07:15 2019 +0100
     1.2 +++ b/Admin/Phabricator/README	Tue Nov 05 22:56:06 2019 +0100
     1.3 @@ -54,11 +54,11 @@
     1.4    Port 222
     1.5  
     1.6    /etc/passwd:
     1.7 -  phab-daemon:x:118:126::/home/phab-daemon:/bin/bash
     1.8 +  phabricator:x:118:126::/home/phabricator:/bin/bash
     1.9    vcs:x:119:125::/home/vcs:/bin/bash
    1.10  
    1.11    /etc/group:
    1.12 -  phab-daemon:x:126:
    1.13 +  phabricator:x:126:
    1.14    vcs:x:125:
    1.15  
    1.16    $ cp ssh/ssh-hook /usr/local/bin/.
    1.17 @@ -66,24 +66,24 @@
    1.18    $ cp ssh/sshd-phabricator.service /lib/systemd/system/.
    1.19    $ cp ssh/sudoers.d/phabricator /etc/sudoers.d/.
    1.20  
    1.21 -  $ ./bin/config set phd.user phab-daemon
    1.22 +  $ ./bin/config set phd.user phabricator
    1.23    $ ./bin/config set diffusion.ssh-user vcs
    1.24    $ ./bin/config set diffusion.ssh-port 22
    1.25  
    1.26 +  $ systemctl enable sshd-phabricator
    1.27    $ systemctl start sshd-phabricator
    1.28 -  $ systemctl enable sshd-phabricator
    1.29  
    1.30    Test on local machine:
    1.31    $ echo "{}" | ssh vcs@phabricator.sketis.net conduit conduit.ping
    1.32  
    1.33  - Repository Local Path:
    1.34      mkdir -p /var/www/phabricator/repo
    1.35 -    chown phab-daemon:phab-daemon /var/www/phabricator/repo
    1.36 +    chown phabricator:phabricator /var/www/phabricator/repo
    1.37  
    1.38  - PHP Daemon:
    1.39    $ cp phd/phd-phabricator.service /lib/systemd/system/.
    1.40 +  $ systemctl enable phd-phabricator
    1.41    $ systemctl start phd-phabricator
    1.42 -  $ systemctl enable phd-phabricator
    1.43  
    1.44  - Update:
    1.45    https://secure.phabricator.com/book/phabricator/article/upgrading
     2.1 --- a/Admin/Phabricator/phd/phd-phabricator.service	Tue Nov 05 21:07:15 2019 +0100
     2.2 +++ b/Admin/Phabricator/phd/phd-phabricator.service	Tue Nov 05 22:56:06 2019 +0100
     2.3 @@ -4,8 +4,8 @@
     2.4  
     2.5  [Service]
     2.6  Type=oneshot
     2.7 -User=phab-daemon
     2.8 -Group=phab-daemon
     2.9 +User=phabricator
    2.10 +Group=phabricator
    2.11  Environment=PATH=/sbin:/usr/sbin:/usr/local/sbin:/usr/local/bin:/usr/bin:/bin
    2.12  ExecStart=/var/www/phabricator/phabricator/bin/phd start
    2.13  ExecStop=/var/www/phabricator/phabricator/bin/phd stop
     3.1 --- a/Admin/Phabricator/ssh/sudoers.d/phabricator	Tue Nov 05 21:07:15 2019 +0100
     3.2 +++ b/Admin/Phabricator/ssh/sudoers.d/phabricator	Tue Nov 05 22:56:06 2019 +0100
     3.3 @@ -1,2 +1,2 @@
     3.4 -www-data ALL=(phab-daemon) SETENV: NOPASSWD: /usr/bin/git, /usr/bin/hg, /usr/bin/ssh, /usr/bin/id
     3.5 -vcs ALL=(phab-daemon) SETENV: NOPASSWD: /usr/bin/git, /usr/bin/git-upload-pack, /usr/bin/git-receive-pack, /usr/bin/hg, /usr/bin/svnserve, /usr/bin/ssh, /usr/bin/id
     3.6 +www-data ALL=(phabricator) SETENV: NOPASSWD: /usr/bin/git, /usr/bin/hg, /usr/bin/ssh, /usr/bin/id
     3.7 +vcs ALL=(phabricator) SETENV: NOPASSWD: /usr/bin/git, /usr/bin/git-upload-pack, /usr/bin/git-receive-pack, /usr/bin/hg, /usr/bin/svnserve, /usr/bin/ssh, /usr/bin/id
     4.1 --- a/etc/options	Tue Nov 05 21:07:15 2019 +0100
     4.2 +++ b/etc/options	Tue Nov 05 22:56:06 2019 +0100
     4.3 @@ -345,8 +345,6 @@
     4.4  
     4.5  section "Phabricator server"
     4.6  
     4.7 -option phabricator_user : string = "phabricator"
     4.8 -
     4.9  option phabricator_www_user : string = "www-data"
    4.10  option phabricator_www_root : string = "/var/www"
    4.11  
     5.1 --- a/src/Pure/System/linux.scala	Tue Nov 05 21:07:15 2019 +0100
     5.2 +++ b/src/Pure/System/linux.scala	Tue Nov 05 22:56:06 2019 +0100
     5.3 @@ -63,4 +63,73 @@
     5.4  
     5.5    def package_install(packages: List[String], progress: Progress = No_Progress): Unit =
     5.6      progress.bash("apt-get install -y -- " + Bash.strings(packages), echo = true).check
     5.7 +
     5.8 +
     5.9 +  /* users */
    5.10 +
    5.11 +  def user_exists(name: String): Boolean =
    5.12 +    Isabelle_System.bash("id " + Bash.string(name)).ok
    5.13 +
    5.14 +  def user_entry(name: String, field: Int): String =
    5.15 +  {
    5.16 +    val result = Isabelle_System.bash("getent passwd " + Bash.string(name)).check
    5.17 +    val fields = space_explode(':', result.out)
    5.18 +
    5.19 +    if (1 <= field && field <= fields.length) fields(field - 1)
    5.20 +    else error("No passwd field " + field + " for user " + quote(name))
    5.21 +  }
    5.22 +
    5.23 +  def user_description(name: String): String = user_entry(name, 5).takeWhile(_ != ',')
    5.24 +
    5.25 +  def user_home(name: String): String = user_entry(name, 6)
    5.26 +
    5.27 +  def user_add(name: String,
    5.28 +    description: String = "",
    5.29 +    system: Boolean = false,
    5.30 +    ssh_setup: Boolean = false)
    5.31 +  {
    5.32 +    require(!description.contains(','))
    5.33 +
    5.34 +    if (user_exists(name)) error("User already exists: " + quote(name))
    5.35 +
    5.36 +    Isabelle_System.bash(
    5.37 +      "adduser --quiet --disabled-password --gecos " + Bash.string(description) +
    5.38 +        (if (system) " --system --group --shell /bin/bash " else "") +
    5.39 +        " " + Bash.string(name)).check
    5.40 +
    5.41 +    if (ssh_setup) {
    5.42 +      val id_rsa = user_home(name) + "/.ssh/id_rsa"
    5.43 +      Isabelle_System.bash("""
    5.44 +if [ ! -f """ + Bash.string(id_rsa) + """ ]
    5.45 +then
    5.46 +  yes '\n' | sudo -i -u """ + Bash.string(name) +
    5.47 +    """ ssh-keygen -q -f """ + Bash.string(id_rsa) + """
    5.48 +fi
    5.49 +      """).check
    5.50 +    }
    5.51 +  }
    5.52 +
    5.53 +
    5.54 +  /* system services */
    5.55 +
    5.56 +  def service_start(name: String): Unit =
    5.57 +    Isabelle_System.bash("systemctl start " + Bash.string(name)).check
    5.58 +
    5.59 +  def service_stop(name: String): Unit =
    5.60 +    Isabelle_System.bash("systemctl stop " + Bash.string(name)).check
    5.61 +
    5.62 +  def service_restart(name: String): Unit =
    5.63 +    Isabelle_System.bash("systemctl restart " + Bash.string(name)).check
    5.64 +
    5.65 +  def service_install(name: String, spec: String)
    5.66 +  {
    5.67 +    val service_file = Path.explode("/lib/systemd/system") + Path.basic(name).ext("service")
    5.68 +    File.write(service_file, spec)
    5.69 +
    5.70 +    Isabelle_System.bash("""
    5.71 +      set -e
    5.72 +      chmod 0644 """ + File.bash_path(service_file) + """
    5.73 +      systemctl enable """ + Bash.string(name) + """
    5.74 +      systemctl start """ + Bash.string(name)).check
    5.75 +  }
    5.76  }
     6.1 --- a/src/Pure/Tools/phabricator.scala	Tue Nov 05 21:07:15 2019 +0100
     6.2 +++ b/src/Pure/Tools/phabricator.scala	Tue Nov 05 22:56:06 2019 +0100
     6.3 @@ -16,15 +16,7 @@
     6.4  {
     6.5    /** defaults **/
     6.6  
     6.7 -  val default_name = "vcs"
     6.8 -
     6.9 -  def default_prefix(name: String): String = "phabricator-" + name
    6.10 -
    6.11 -  def default_root(options: Options, name: String): Path =
    6.12 -    Path.explode(options.string("phabricator_www_root")) + Path.basic(default_prefix(name))
    6.13 -
    6.14 -  def default_repo(options: Options, name: String): Path =
    6.15 -    default_root(options, name) + Path.basic("repo")
    6.16 +  /* required packages */
    6.17  
    6.18    val packages: List[String] =
    6.19      Build_Docker.packages :::
    6.20 @@ -33,17 +25,44 @@
    6.21        "git", "mysql-server", "apache2", "libapache2-mod-php", "php", "php-mysql",
    6.22        "php-gd", "php-curl", "php-apcu", "php-cli", "php-json", "php-mbstring",
    6.23        // more packages
    6.24 -      "php-zip", "python-pygments")
    6.25 +      "php-zip", "python-pygments", "ssh")
    6.26 +
    6.27 +
    6.28 +  /* global system resources */
    6.29 +
    6.30 +  val daemon_user = "phabricator"
    6.31 +
    6.32 +  val ssh_standard = 22
    6.33 +  val ssh_alternative1 = 222
    6.34 +  val ssh_alternative2 = 2222
    6.35 +
    6.36 +
    6.37 +  /* installation parameters */
    6.38 +
    6.39 +  val default_name = "vcs"
    6.40 +
    6.41 +  def phabricator_name(name: String = "", ext: String = ""): String =
    6.42 +    "phabricator" + (if (name.isEmpty) "" else "-" + name) + (if (ext.isEmpty) "" else "." + ext)
    6.43 +
    6.44 +  def isabelle_phabricator_name(name: String = "", ext: String = ""): String =
    6.45 +    "isabelle-" + phabricator_name(name = name, ext = ext)
    6.46 +
    6.47 +  def default_root(options: Options, name: String): Path =
    6.48 +    Path.explode(options.string("phabricator_www_root")) +
    6.49 +    Path.basic(phabricator_name(name = name))
    6.50 +
    6.51 +  def default_repo(options: Options, name: String): Path =
    6.52 +    default_root(options, name) + Path.basic("repo")
    6.53  
    6.54  
    6.55  
    6.56    /** global configuration **/
    6.57  
    6.58 -  val global_config = Path.explode("/etc/isabelle-phabricator.conf")
    6.59 +  val global_config = Path.explode("/etc/" + isabelle_phabricator_name(ext = "conf"))
    6.60  
    6.61    sealed case class Config(name: String, root: Path)
    6.62    {
    6.63 -    def home: Path = root + Path.explode("phabricator")
    6.64 +    def home: Path = root + Path.explode(phabricator_name())
    6.65  
    6.66      def execute(command: String): Process_Result =
    6.67        Isabelle_System.bash("./bin/" + command, cwd = home.file).check
    6.68 @@ -77,28 +96,52 @@
    6.69  
    6.70    /** setup **/
    6.71  
    6.72 +  def user_setup(name: String, description: String, ssh_setup: Boolean = false)
    6.73 +  {
    6.74 +    if (!Linux.user_exists(name)) {
    6.75 +      Linux.user_add(name, description = description, system = true, ssh_setup = ssh_setup)
    6.76 +    }
    6.77 +    else if (Linux.user_description(name) != description) {
    6.78 +      error("User " + quote(name) + " already exists --" +
    6.79 +        " for Phabricator it should have the description:\n  " + quote(description))
    6.80 +    }
    6.81 +  }
    6.82 +
    6.83    def phabricator_setup(
    6.84      options: Options,
    6.85      name: String = default_name,
    6.86 -    prefix: String = "",
    6.87      root: String = "",
    6.88      repo: String = "",
    6.89 +    package_update: Boolean = false,
    6.90      progress: Progress = No_Progress)
    6.91    {
    6.92      /* system environment */
    6.93  
    6.94      Linux.check_system_root()
    6.95  
    6.96 -    Linux.package_update(progress = progress)
    6.97 -    Linux.check_reboot_required()
    6.98 +    if (package_update) {
    6.99 +      Linux.package_update(progress = progress)
   6.100 +      Linux.check_reboot_required()
   6.101 +    }
   6.102  
   6.103      Linux.package_install(packages, progress = progress)
   6.104      Linux.check_reboot_required()
   6.105  
   6.106  
   6.107 +    /* users */
   6.108 +
   6.109 +    if (name == daemon_user) {
   6.110 +      error("Clash of installation name with daemon user " + quote(daemon_user))
   6.111 +    }
   6.112 +
   6.113 +    user_setup(daemon_user, "Phabricator Daemon User", ssh_setup = true)
   6.114 +    user_setup(name, "Phabricator SSH User")
   6.115 +
   6.116 +    val www_user = options.string("phabricator_www_user")
   6.117 +
   6.118 +
   6.119      /* basic installation */
   6.120  
   6.121 -    val prefix_name = proper_string(prefix) getOrElse default_prefix(name)
   6.122      val root_path = if (root.nonEmpty) Path.explode(root) else default_root(options, name)
   6.123      val repo_path = if (repo.nonEmpty) Path.explode(repo) else default_repo(options, name)
   6.124  
   6.125 @@ -115,7 +158,7 @@
   6.126      progress.bash(cwd = root_path.file, echo = true,
   6.127        script = """
   6.128          set -e
   6.129 -        chown """ + Bash.string(options.string("phabricator_www_user")) + """ .
   6.130 +        chown """ + Bash.string(www_user) + ":" + Bash.string(www_user) + """ .
   6.131          chmod 755 .
   6.132  
   6.133          git clone https://github.com/phacility/libphutil.git
   6.134 @@ -126,11 +169,39 @@
   6.135      val config = Config(name, root_path)
   6.136      write_config(configs ::: List(config))
   6.137  
   6.138 +    config.execute("config set pygments.enabled true")
   6.139 +
   6.140 +
   6.141 +    /* local repository directory */
   6.142 +
   6.143 +    if (!Isabelle_System.bash("mkdir -p " + File.bash_path(repo_path)).ok) {
   6.144 +      error("Failed to create local repository directory " + repo_path)
   6.145 +    }
   6.146 +
   6.147 +    Isabelle_System.bash(cwd = repo_path.file,
   6.148 +      script = """
   6.149 +        set -e
   6.150 +        chown -R """ + Bash.string(daemon_user) + ":" + Bash.string(daemon_user) + """ .
   6.151 +        chmod 755 .
   6.152 +      """).check
   6.153 +
   6.154 +    config.execute("config set repository.default-local-path " + File.bash_path(repo_path))
   6.155 +
   6.156  
   6.157      /* MySQL setup */
   6.158  
   6.159      progress.echo("MySQL setup...")
   6.160  
   6.161 +    File.write(Path.explode("/etc/mysql/mysql.conf.d/" + phabricator_name(ext = "cnf")),
   6.162 +"""[mysqld]
   6.163 +max_allowed_packet = 32M
   6.164 +innodb_buffer_pool_size = 1600M
   6.165 +local_infile = 0
   6.166 +""")
   6.167 +
   6.168 +    Linux.service_restart("mysql")
   6.169 +
   6.170 +
   6.171      def mysql_conf(R: Regex): Option[String] =
   6.172        split_lines(File.read(Path.explode(options.string("phabricator_mysql_config")))).
   6.173          collectFirst({ case R(a) => a })
   6.174 @@ -144,11 +215,47 @@
   6.175      }
   6.176  
   6.177      config.execute("config set storage.default-namespace " +
   6.178 -      Bash.string(prefix_name.replace("-", "_")))
   6.179 +      Bash.string(phabricator_name(name = name).replace("-", "_")))
   6.180 +
   6.181 +    config.execute("config set storage.mysql-engine.max-size 8388608")
   6.182  
   6.183      config.execute("storage upgrade --force")
   6.184  
   6.185  
   6.186 +    /* SSH hosting */
   6.187 +
   6.188 +    progress.echo("SSH hosting setup...")
   6.189 +
   6.190 +    val ssh_port = ssh_alternative2
   6.191 +
   6.192 +    config.execute("config set diffusion.ssh-user " + Bash.string(name))
   6.193 +    config.execute("config set diffusion.ssh-port " + ssh_port)
   6.194 +
   6.195 +    val sudoers_file = Path.explode("/etc/sudoers.d") + Path.basic(isabelle_phabricator_name())
   6.196 +    File.write(sudoers_file,
   6.197 +      www_user + " ALL=(" + daemon_user + ") SETENV: NOPASSWD: /usr/bin/git, /usr/bin/hg, /usr/bin/ssh, /usr/bin/id\n" +
   6.198 +      name + " ALL=(" + daemon_user + ") SETENV: NOPASSWD: /usr/bin/git, /usr/bin/git-upload-pack, /usr/bin/git-receive-pack, /usr/bin/hg, /usr/bin/svnserve, /usr/bin/ssh, /usr/bin/id\n")
   6.199 +
   6.200 +    Isabelle_System.bash("chmod 0440 " + File.bash_path(sudoers_file)).check
   6.201 +
   6.202 +
   6.203 +    /* PHP setup */
   6.204 +
   6.205 +    val php_version =
   6.206 +      Isabelle_System.bash("""php --run 'echo PHP_MAJOR_VERSION . "." . PHP_MINOR_VERSION;'""")
   6.207 +        .check.out
   6.208 +
   6.209 +    val php_conf =
   6.210 +      Path.explode("/etc/php") + Path.basic(php_version) +  // educated guess
   6.211 +        Path.explode("apache2/conf.d") +
   6.212 +        Path.basic(isabelle_phabricator_name(ext = "ini"))
   6.213 +
   6.214 +    File.write(php_conf,
   6.215 +      "post_max_size = 32M\n" +
   6.216 +      "opcache.validate_timestamps = 0\n" +
   6.217 +      "memory_limit = 512M\n")
   6.218 +
   6.219 +
   6.220      /* Apache setup */
   6.221  
   6.222      progress.echo("Apache setup...")
   6.223 @@ -158,10 +265,12 @@
   6.224  
   6.225      if (!apache_sites.is_dir) error("Bad Apache sites directory " + apache_sites)
   6.226  
   6.227 -    File.write(apache_sites + Path.basic(prefix_name + ".conf"),
   6.228 +    val server_name = phabricator_name(name = name, ext = "lvh.me")  // alias for "localhost" for testing
   6.229 +    val server_url = "http://" + server_name
   6.230 +
   6.231 +    File.write(apache_sites + Path.basic(isabelle_phabricator_name(name = name, ext = "conf")),
   6.232  """<VirtualHost *:80>
   6.233 -    #ServerName: "lvh.me" is an alias for "localhost" for testing
   6.234 -    ServerName """ + prefix_name + """.lvh.me
   6.235 +    ServerName """ + server_name + """
   6.236      ServerAdmin webmaster@localhost
   6.237      DocumentRoot """ + config.home.implode + """/webroot
   6.238  
   6.239 @@ -173,14 +282,42 @@
   6.240  # vim: syntax=apache ts=4 sw=4 sts=4 sr noet
   6.241  """)
   6.242  
   6.243 -    Isabelle_System.bash("""
   6.244 +    Isabelle_System.bash( """
   6.245        set -e
   6.246        a2enmod rewrite
   6.247 -      a2ensite """ + Bash.string(prefix_name) + """
   6.248 -      systemctl restart apache2
   6.249 -""").check
   6.250 +      a2ensite """ + Bash.string(isabelle_phabricator_name(name = name))).check
   6.251 +
   6.252 +    config.execute("config set phabricator.base-uri " + Bash.string(server_url))
   6.253 +
   6.254 +    Linux.service_restart("apache2")
   6.255 +
   6.256 +
   6.257 +    /* PHP daemon */
   6.258 +
   6.259 +    progress.echo("PHP daemon setup...")
   6.260 +
   6.261 +    config.execute("config set phd.user " + Bash.string(daemon_user))
   6.262  
   6.263 -    progress.echo("\nDONE\nWeb configuration via http://" + prefix_name + ".lvh.me")
   6.264 +    Linux.service_install(isabelle_phabricator_name(name = name),
   6.265 +"""[Unit]
   6.266 +Description=PHP daemon for Isabelle/Phabricator """ + quote(name) + """
   6.267 +After=syslog.target network.target apache2.service mysql.service
   6.268 +
   6.269 +[Service]
   6.270 +Type=oneshot
   6.271 +User=""" + daemon_user + """
   6.272 +Group=""" + daemon_user + """
   6.273 +Environment=PATH=/sbin:/usr/sbin:/usr/local/sbin:/usr/local/bin:/usr/bin:/bin
   6.274 +ExecStart=""" + config.home.implode + """/bin/phd start
   6.275 +ExecStop=""" + config.home.implode + """/bin/phd stop
   6.276 +RemainAfterExit=yes
   6.277 +
   6.278 +[Install]
   6.279 +WantedBy=multi-user.target
   6.280 +""")
   6.281 +
   6.282 +
   6.283 +    progress.echo("\nDONE\nWeb configuration via " + server_url)
   6.284    }
   6.285  
   6.286  
   6.287 @@ -189,10 +326,10 @@
   6.288    val isabelle_tool1 =
   6.289      Isabelle_Tool("phabricator_setup", "setup Phabricator server on Ubuntu Linux", args =>
   6.290      {
   6.291 +      var repo = ""
   6.292 +      var package_update = false
   6.293        var options = Options.init()
   6.294 -      var prefix = ""
   6.295        var root = ""
   6.296 -      var repo = ""
   6.297  
   6.298        val getopts =
   6.299          Getopts("""
   6.300 @@ -200,8 +337,8 @@
   6.301  
   6.302    Options are:
   6.303      -R DIR       repository directory (default: """ + default_repo(options, "NAME") + """)
   6.304 +    -U           full update of system packages before installation
   6.305      -o OPTION    override Isabelle system OPTION (via NAME=VAL or NAME)
   6.306 -    -p PREFIX    prefix for derived names (default: """ + default_prefix("NAME") + """)
   6.307      -r DIR       installation root directory (default: """ + default_root(options, "NAME") + """)
   6.308  
   6.309    Install Phabricator as Ubuntu LAMP application (Linux, Apache, MySQL, PHP).
   6.310 @@ -213,8 +350,8 @@
   6.311    a regular Unix user and used for public SSH access.
   6.312  """,
   6.313            "R:" -> (arg => repo = arg),
   6.314 +          "U" -> (_ => package_update = true),
   6.315            "o:" -> (arg => options = options + arg),
   6.316 -          "p:" -> (arg => prefix = arg),
   6.317            "r:" -> (arg => root = arg))
   6.318  
   6.319        val more_args = getopts(args)
   6.320 @@ -228,8 +365,8 @@
   6.321  
   6.322        val progress = new Console_Progress
   6.323  
   6.324 -      phabricator_setup(options, name, prefix = prefix, root = root, repo = repo,
   6.325 -        progress = progress)
   6.326 +      phabricator_setup(options, name, root = root, repo = repo,
   6.327 +        package_update = package_update, progress = progress)
   6.328      })
   6.329  
   6.330