src/Pure/Build/build.scala
changeset 80196 9308bc5f65d6
parent 80160 ead20482da9c
child 80350 96843eb96493
equal deleted inserted replaced
80195:e2ccabd7a857 80196:9308bc5f65d6
   597     host: Build_Cluster.Host,
   597     host: Build_Cluster.Host,
   598     ssh: SSH.System = SSH.Local,
   598     ssh: SSH.System = SSH.Local,
   599     build_options: List[Options.Spec] = Nil,
   599     build_options: List[Options.Spec] = Nil,
   600     build_id: String = "",
   600     build_id: String = "",
   601     isabelle_home: Path = Path.current,
   601     isabelle_home: Path = Path.current,
   602     afp_root: Option[Path] = None,
       
   603     dirs: List[Path] = Nil,
   602     dirs: List[Path] = Nil,
   604     quiet: Boolean = false,
   603     quiet: Boolean = false,
   605     verbose: Boolean = false
   604     verbose: Boolean = false
   606   ): String = {
   605   ): String = {
   607     val options = build_options ::: Options.Spec.eq("build_hostname", host.name) :: host.options
   606     val options = build_options ::: Options.Spec.eq("build_hostname", host.name) :: host.options
   608     ssh.bash_path(Isabelle_Tool.exe(isabelle_home)) + " build_worker" +
   607     ssh.bash_path(Isabelle_Tool.exe(isabelle_home)) + " build_worker" +
   609       if_proper(build_id, " -B " + Bash.string(build_id)) +
   608       if_proper(build_id, " -B " + Bash.string(build_id)) +
   610       if_proper(afp_root, " -A " + ssh.bash_path(afp_root.get)) +
       
   611       dirs.map(dir => " -d " + ssh.bash_path(dir)).mkString +
   609       dirs.map(dir => " -d " + ssh.bash_path(dir)).mkString +
   612       if_proper(host.numa, " -N") + " -j" + host.jobs +
   610       if_proper(host.numa, " -N") + " -j" + host.jobs +
   613       Options.Spec.bash_strings(options, bg = true) +
   611       Options.Spec.bash_strings(options, bg = true) +
   614       if_proper(quiet, " -q") +
   612       if_proper(quiet, " -q") +
   615       if_proper(verbose, " -v")
   613       if_proper(verbose, " -v")
   617 
   615 
   618   def build_worker(
   616   def build_worker(
   619     options: Options,
   617     options: Options,
   620     build_id: String = "",
   618     build_id: String = "",
   621     progress: Progress = new Progress,
   619     progress: Progress = new Progress,
   622     afp_root: Option[Path] = None,
       
   623     dirs: List[Path] = Nil,
   620     dirs: List[Path] = Nil,
   624     numa_shuffling: Boolean = false,
   621     numa_shuffling: Boolean = false,
   625     max_jobs: Option[Int] = None
   622     max_jobs: Option[Int] = None
   626   ): Results = {
   623   ): Results = {
   627     val engine = Engine(engine_name(options))
   624     val engine = Engine(engine_name(options))
   632       using_optional(store.maybe_open_build_database(server = server)) { build_database =>
   629       using_optional(store.maybe_open_build_database(server = server)) { build_database =>
   633         val builds = read_builds(build_database)
   630         val builds = read_builds(build_database)
   634 
   631 
   635         val build_master = find_builds(build_database, build_id, builds.filter(_.active))
   632         val build_master = find_builds(build_database, build_id, builds.filter(_.active))
   636 
   633 
       
   634         val more_dirs = List(Path.ISABELLE_HOME + Sync.DIRS).filter(Sessions.is_session_dir(_))
       
   635 
   637         val sessions_structure =
   636         val sessions_structure =
   638           Sessions.load_structure(build_options, dirs = AFP.main_dirs(afp_root) ::: dirs).
   637           Sessions.load_structure(build_options, dirs = more_dirs ::: dirs).
   639             selection(Sessions.Selection(sessions = build_master.sessions))
   638             selection(Sessions.Selection(sessions = build_master.sessions))
   640 
   639 
   641         val build_deps =
   640         val build_deps =
   642           Sessions.deps(sessions_structure, progress = progress, inlined_files = true).check_errors
   641           Sessions.deps(sessions_structure, progress = progress, inlined_files = true).check_errors
   643 
   642 
   644         val build_context =
   643         val build_context =
   645           Context(store, build_deps, engine = engine, afp_root = afp_root,
   644           Context(store, build_deps, engine = engine, hostname = hostname(build_options),
   646             hostname = hostname(build_options), numa_shuffling = numa_shuffling,
   645             numa_shuffling = numa_shuffling, build_uuid = build_master.build_uuid,
   647             build_uuid = build_master.build_uuid, build_start = Some(build_master.start),
   646             build_start = Some(build_master.start), jobs = max_jobs.getOrElse(1))
   648             jobs = max_jobs.getOrElse(1))
       
   649 
   647 
   650         engine.run_build_process(build_context, progress, server)
   648         engine.run_build_process(build_context, progress, server)
   651       }
   649       }
   652     }
   650     }
   653   }
   651   }
   654 
   652 
   655   val isabelle_tool3 = Isabelle_Tool("build_worker", "start worker for session build process",
   653   val isabelle_tool3 = Isabelle_Tool("build_worker", "start worker for session build process",
   656     Scala_Project.here,
   654     Scala_Project.here,
   657     { args =>
   655     { args =>
   658       var afp_root: Option[Path] = None
       
   659       var build_id = ""
   656       var build_id = ""
   660       var numa_shuffling = false
   657       var numa_shuffling = false
   661       val dirs = new mutable.ListBuffer[Path]
   658       val dirs = new mutable.ListBuffer[Path]
   662       var max_jobs: Option[Int] = None
   659       var max_jobs: Option[Int] = None
   663       var options =
   660       var options =
   668 
   665 
   669       val getopts = Getopts("""
   666       val getopts = Getopts("""
   670 Usage: isabelle build_worker [OPTIONS]
   667 Usage: isabelle build_worker [OPTIONS]
   671 
   668 
   672   Options are:
   669   Options are:
   673     -A ROOT      include AFP with given root directory (":" for """ + AFP.BASE.implode + """)
       
   674     -B UUID      existing UUID for build process (default: from database)
   670     -B UUID      existing UUID for build process (default: from database)
   675     -N           cyclic shuffling of NUMA CPU nodes (performance tuning)
   671     -N           cyclic shuffling of NUMA CPU nodes (performance tuning)
   676     -d DIR       include session directory
   672     -d DIR       include session directory
   677     -j INT       maximum number of parallel jobs (default 1)
   673     -j INT       maximum number of parallel jobs (default 1)
   678     -o OPTION    override Isabelle system OPTION (via NAME=VAL or NAME)
   674     -o OPTION    override Isabelle system OPTION (via NAME=VAL or NAME)
   679     -q           quiet mode: no progress
   675     -q           quiet mode: no progress
   680     -v           verbose
   676     -v           verbose
   681 """,
   677 """,
   682         "A:" -> (arg => afp_root = Some(if (arg == ":") AFP.BASE else Path.explode(arg))),
       
   683         "B:" -> (arg => build_id = arg),
   678         "B:" -> (arg => build_id = arg),
   684         "N" -> (_ => numa_shuffling = true),
   679         "N" -> (_ => numa_shuffling = true),
   685         "d:" -> (arg => dirs += Path.explode(arg)),
   680         "d:" -> (arg => dirs += Path.explode(arg)),
   686         "j:" -> (arg => max_jobs = Some(Value.Nat.parse(arg))),
   681         "j:" -> (arg => max_jobs = Some(Value.Nat.parse(arg))),
   687         "o:" -> (arg => options = options + arg),
   682         "o:" -> (arg => options = options + arg),
   699       val results =
   694       val results =
   700         progress.interrupt_handler {
   695         progress.interrupt_handler {
   701           build_worker(options,
   696           build_worker(options,
   702             build_id = build_id,
   697             build_id = build_id,
   703             progress = progress,
   698             progress = progress,
   704             afp_root = afp_root,
       
   705             dirs = dirs.toList,
   699             dirs = dirs.toList,
   706             numa_shuffling = Host.numa_check(progress, numa_shuffling),
   700             numa_shuffling = Host.numa_check(progress, numa_shuffling),
   707             max_jobs = max_jobs)
   701             max_jobs = max_jobs)
   708         }
   702         }
   709 
   703