Rename output logs file and agent's output trajectory file (#230)

* Pull changes from #107 and #212

* Add a workspace class. Refactoring + tests

* Refactor SWE-Bench and make sure all 500 tasks can be solved.

* Rename output logs file and agent's output trajectory file

* Pass revision id to parent class

* Tests should check for default SWE-bench/SWE-bench_Verified

* Make sure we apply any changes needed for setting up the environment.

* Rename RemoteWorkspace -> Workspace

* Fix typos.

* Use better delimiter for here-document + add explantion to workspace.write_file

* Show with hidden files when listing file with workspace

* Fix interact_with_pdb test

* Raises if git diff fails

* Rename log to trajectory

* fix test, now log file is named debug_gym.log

---------

Co-authored-by: Matheus Pereira <matpereira@microsoft.com>
Co-authored-by: Xingdi (Eric) Yuan <xingdi-eric-yuan@users.noreply.github.com>

Author: 3 people

debug_gym/agents/base_agent.py

@@ -347,10 +347,10 @@ def save_patch(self, task_name="custom"):
             f"Patch saved in {pjoin(self._output_path, task_name, 'debug_gym.patch')}"
         )
 
-    def log(self, task_name="custom"):
+    def save_trajectory(self, task_name="custom"):
         # Simple tools list.
         tools = [f"{tool.name}({tool.arguments})" for tool in self.env.tools]
-        jsonl_output = {
+        json_output = {
             "problem": task_name,
             "config": self.config,
             "tools": self.llm.define_tools(self.env.tools) if self.llm else tools,
@@ -362,14 +362,13 @@ def log(self, task_name="custom"):
         }
         for step_id in range(len(self.history)):
             step_json = self.history.json(step_id)
-            jsonl_output["log"].append(step_json)
+            json_output["log"].append(step_json)
         os.makedirs(pjoin(self._output_path, task_name), exist_ok=True)
-        with open(pjoin(self._output_path, task_name, "debug_gym.jsonl"), "w") as f:
-            json.dump(jsonl_output, f, indent=4)
+        json_file = pjoin(self._output_path, task_name, "trajectory.json")
+        with open(json_file, "w") as f:
+            json.dump(json_output, f, indent=4)
 
-        self.logger.debug(
-            f"Log saved in {pjoin(self._output_path, task_name, 'debug_gym.jsonl')}"
-        )
+        self.logger.debug(f"Trajectory saved in {json_file}")
 
 
 def create_agent(agent_type: str, **agent_kwargs):

debug_gym/logger.py

@@ -109,7 +109,7 @@ def log_file_path(self) -> str:
             str(
                 log_file_path(
                     self.logdir,
-                    self.problem_id,
+                    "debug_gym",
                     relative=True,
                 )
             )
@@ -551,7 +551,7 @@ def _initialize_main_logger(self, level):
 
     def _initialize_file_handler(self, name: str, mode: str):
         self.setLevel(logging.DEBUG)  # Ensure logger operates at DEBUG level
-        self.log_file = log_file_path(self.log_dir, name)
+        self.log_file = log_file_path(self.log_dir, "debug_gym")
         fh = logging.FileHandler(self.log_file, mode=mode)
         formatter = StripAnsiFormatter("%(asctime)s %(levelname)-8s %(message)s")
         fh.setFormatter(formatter)

scripts/run.py

@@ -139,8 +139,8 @@ def run_agent(args, problem, config):
             report_progress_error = False
             raise
 
-        # save log
-        agent.log(task_name=problem)
+        # save trajectory
+        agent.save_trajectory(task_name=problem)
 
         # optionally apply patch
         if config["save_patch"]:

tests/test_logger.py

@@ -157,7 +157,7 @@ def test_task_log_file_path(tmp_path):
         status="pending",
         logdir=str(log_dir),
     )
-    expected_path = str(log_dir / "test_task.log")
+    expected_path = str(log_dir / "debug_gym.log")
     assert task.log_file_path == expected_path