597 host: Build_Cluster.Host, |
597 host: Build_Cluster.Host, |
598 ssh: SSH.System = SSH.Local, |
598 ssh: SSH.System = SSH.Local, |
599 build_options: List[Options.Spec] = Nil, |
599 build_options: List[Options.Spec] = Nil, |
600 build_id: String = "", |
600 build_id: String = "", |
601 isabelle_home: Path = Path.current, |
601 isabelle_home: Path = Path.current, |
602 afp_root: Option[Path] = None, |
|
603 dirs: List[Path] = Nil, |
602 dirs: List[Path] = Nil, |
604 quiet: Boolean = false, |
603 quiet: Boolean = false, |
605 verbose: Boolean = false |
604 verbose: Boolean = false |
606 ): String = { |
605 ): String = { |
607 val options = build_options ::: Options.Spec.eq("build_hostname", host.name) :: host.options |
606 val options = build_options ::: Options.Spec.eq("build_hostname", host.name) :: host.options |
608 ssh.bash_path(Isabelle_Tool.exe(isabelle_home)) + " build_worker" + |
607 ssh.bash_path(Isabelle_Tool.exe(isabelle_home)) + " build_worker" + |
609 if_proper(build_id, " -B " + Bash.string(build_id)) + |
608 if_proper(build_id, " -B " + Bash.string(build_id)) + |
610 if_proper(afp_root, " -A " + ssh.bash_path(afp_root.get)) + |
|
611 dirs.map(dir => " -d " + ssh.bash_path(dir)).mkString + |
609 dirs.map(dir => " -d " + ssh.bash_path(dir)).mkString + |
612 if_proper(host.numa, " -N") + " -j" + host.jobs + |
610 if_proper(host.numa, " -N") + " -j" + host.jobs + |
613 Options.Spec.bash_strings(options, bg = true) + |
611 Options.Spec.bash_strings(options, bg = true) + |
614 if_proper(quiet, " -q") + |
612 if_proper(quiet, " -q") + |
615 if_proper(verbose, " -v") |
613 if_proper(verbose, " -v") |
617 |
615 |
618 def build_worker( |
616 def build_worker( |
619 options: Options, |
617 options: Options, |
620 build_id: String = "", |
618 build_id: String = "", |
621 progress: Progress = new Progress, |
619 progress: Progress = new Progress, |
622 afp_root: Option[Path] = None, |
|
623 dirs: List[Path] = Nil, |
620 dirs: List[Path] = Nil, |
624 numa_shuffling: Boolean = false, |
621 numa_shuffling: Boolean = false, |
625 max_jobs: Option[Int] = None |
622 max_jobs: Option[Int] = None |
626 ): Results = { |
623 ): Results = { |
627 val engine = Engine(engine_name(options)) |
624 val engine = Engine(engine_name(options)) |
632 using_optional(store.maybe_open_build_database(server = server)) { build_database => |
629 using_optional(store.maybe_open_build_database(server = server)) { build_database => |
633 val builds = read_builds(build_database) |
630 val builds = read_builds(build_database) |
634 |
631 |
635 val build_master = find_builds(build_database, build_id, builds.filter(_.active)) |
632 val build_master = find_builds(build_database, build_id, builds.filter(_.active)) |
636 |
633 |
|
634 val more_dirs = List(Path.ISABELLE_HOME + Sync.DIRS).filter(Sessions.is_session_dir(_)) |
|
635 |
637 val sessions_structure = |
636 val sessions_structure = |
638 Sessions.load_structure(build_options, dirs = AFP.main_dirs(afp_root) ::: dirs). |
637 Sessions.load_structure(build_options, dirs = more_dirs ::: dirs). |
639 selection(Sessions.Selection(sessions = build_master.sessions)) |
638 selection(Sessions.Selection(sessions = build_master.sessions)) |
640 |
639 |
641 val build_deps = |
640 val build_deps = |
642 Sessions.deps(sessions_structure, progress = progress, inlined_files = true).check_errors |
641 Sessions.deps(sessions_structure, progress = progress, inlined_files = true).check_errors |
643 |
642 |
644 val build_context = |
643 val build_context = |
645 Context(store, build_deps, engine = engine, afp_root = afp_root, |
644 Context(store, build_deps, engine = engine, hostname = hostname(build_options), |
646 hostname = hostname(build_options), numa_shuffling = numa_shuffling, |
645 numa_shuffling = numa_shuffling, build_uuid = build_master.build_uuid, |
647 build_uuid = build_master.build_uuid, build_start = Some(build_master.start), |
646 build_start = Some(build_master.start), jobs = max_jobs.getOrElse(1)) |
648 jobs = max_jobs.getOrElse(1)) |
|
649 |
647 |
650 engine.run_build_process(build_context, progress, server) |
648 engine.run_build_process(build_context, progress, server) |
651 } |
649 } |
652 } |
650 } |
653 } |
651 } |
654 |
652 |
655 val isabelle_tool3 = Isabelle_Tool("build_worker", "start worker for session build process", |
653 val isabelle_tool3 = Isabelle_Tool("build_worker", "start worker for session build process", |
656 Scala_Project.here, |
654 Scala_Project.here, |
657 { args => |
655 { args => |
658 var afp_root: Option[Path] = None |
|
659 var build_id = "" |
656 var build_id = "" |
660 var numa_shuffling = false |
657 var numa_shuffling = false |
661 val dirs = new mutable.ListBuffer[Path] |
658 val dirs = new mutable.ListBuffer[Path] |
662 var max_jobs: Option[Int] = None |
659 var max_jobs: Option[Int] = None |
663 var options = |
660 var options = |
668 |
665 |
669 val getopts = Getopts(""" |
666 val getopts = Getopts(""" |
670 Usage: isabelle build_worker [OPTIONS] |
667 Usage: isabelle build_worker [OPTIONS] |
671 |
668 |
672 Options are: |
669 Options are: |
673 -A ROOT include AFP with given root directory (":" for """ + AFP.BASE.implode + """) |
|
674 -B UUID existing UUID for build process (default: from database) |
670 -B UUID existing UUID for build process (default: from database) |
675 -N cyclic shuffling of NUMA CPU nodes (performance tuning) |
671 -N cyclic shuffling of NUMA CPU nodes (performance tuning) |
676 -d DIR include session directory |
672 -d DIR include session directory |
677 -j INT maximum number of parallel jobs (default 1) |
673 -j INT maximum number of parallel jobs (default 1) |
678 -o OPTION override Isabelle system OPTION (via NAME=VAL or NAME) |
674 -o OPTION override Isabelle system OPTION (via NAME=VAL or NAME) |
679 -q quiet mode: no progress |
675 -q quiet mode: no progress |
680 -v verbose |
676 -v verbose |
681 """, |
677 """, |
682 "A:" -> (arg => afp_root = Some(if (arg == ":") AFP.BASE else Path.explode(arg))), |
|
683 "B:" -> (arg => build_id = arg), |
678 "B:" -> (arg => build_id = arg), |
684 "N" -> (_ => numa_shuffling = true), |
679 "N" -> (_ => numa_shuffling = true), |
685 "d:" -> (arg => dirs += Path.explode(arg)), |
680 "d:" -> (arg => dirs += Path.explode(arg)), |
686 "j:" -> (arg => max_jobs = Some(Value.Nat.parse(arg))), |
681 "j:" -> (arg => max_jobs = Some(Value.Nat.parse(arg))), |
687 "o:" -> (arg => options = options + arg), |
682 "o:" -> (arg => options = options + arg), |