Bxyu/miniswe configs

nemo_gym/global_config.py

@@ -15,7 +15,7 @@
 from os import getenv
 from pathlib import Path
 from platform import python_version
-from socket import socket
+from socket import gethostbyname, gethostname, socket
 from typing import ClassVar, List, Optional, Tuple, Type
 
 import hydra
@@ -44,6 +44,7 @@
 DISALLOWED_PORTS_KEY_NAME = "disallowed_ports"
 HEAD_SERVER_DEPS_KEY_NAME = "head_server_deps"
 PYTHON_VERSION_KEY_NAME = "python_version"
+USE_ABSOLUTE_IP = "use_absolute_ip"
 NEMO_GYM_RESERVED_TOP_LEVEL_KEYS = [
     CONFIG_PATHS_KEY_NAME,
     ENTRYPOINT_KEY_NAME,
@@ -52,6 +53,7 @@
     DISALLOWED_PORTS_KEY_NAME,
     HEAD_SERVER_DEPS_KEY_NAME,
     PYTHON_VERSION_KEY_NAME,
+    USE_ABSOLUTE_IP,
 ]
 
 POLICY_BASE_URL_KEY_NAME = "policy_base_url"
@@ -229,9 +231,14 @@ def parse(self, parse_config: Optional[GlobalConfigDictParserConfig] = None) ->
 
         server_instance_configs = self.filter_for_server_instance_configs(global_config_dict)
 
-        # Do one pass through all the configs validate and populate various configs for our servers.
-        default_host = global_config_dict.get(DEFAULT_HOST_KEY_NAME) or "127.0.0.1"
+        use_absolute_ip = global_config_dict.get(USE_ABSOLUTE_IP, False)
+
+        if use_absolute_ip:
+            default_host = gethostbyname(gethostname())
+        else:
+            default_host = global_config_dict.get(DEFAULT_HOST_KEY_NAME) or "127.0.0.1"
 
+        # Do one pass through all the configs validate and populate various configs for our servers.
         head_server_config = global_config_dict.get(HEAD_SERVER_KEY_NAME, {})
         head_server_port = head_server_config.get("port", DEFAULT_HEAD_SERVER_PORT)
 

resources_servers/mini_swe_resource/README.md

@@ -1,6 +1,12 @@
 # Description
 
-Data links: ?
+```bash
+config_paths="responses_api_models/openai_model/configs/openai_model.yaml,\
+resources_servers/mini_swe_resource/configs/mini_swe_resource.yaml"
+ng_run "+config_paths=[$config_paths]" \
+    ++mini_swe_main_agent_train.responses_api_agents.mini_swe_agent.cache_dir_template={your cache dir} \
+    ++mini_swe_main_agent_validation.responses_api_agents.mini_swe_agent.cache_dir_template={your cache dir}
+```
 
 # Licensing information
 Code: ?

resources_servers/mini_swe_resource/configs/mini_swe_resource.yaml

@@ -1,10 +1,11 @@
+# This resources server is unused. It's just here as a placeholder for meta information
 mini_swe_resource_resources_server:
   resources_servers:
     mini_swe_resource:
       entrypoint: app.py
       domain: coding
       verified: false
-mini_swe_simple_agent:
+mini_swe_main_agent_train:
   responses_api_agents:
     mini_swe_agent:
       entrypoint: app.py
@@ -23,6 +24,30 @@ mini_swe_simple_agent:
           version: 0.0.1
           artifact_fpath: train.jsonl
         license: MIT
+        num_repeats: 100  # Just some big repeat number
+      - name: example
+        type: example
+        jsonl_fpath: resources_servers/mini_swe_resource/data/example.jsonl
+      concurrency: 16
+      env: singularity
+      cache_dir_template: ???
+      run_golden: False
+      step_timeout: 300
+      eval_timeout: 900
+      skip_if_exists: False
+      step_limit: 150
+      collapse_limit: 3
+mini_swe_main_agent_validation:
+  responses_api_agents:
+    mini_swe_agent:
+      entrypoint: app.py
+      resources_server:
+        type: resources_servers
+        name: mini_swe_resource_resources_server
+      model_server:
+        type: responses_api_models
+        name: policy_model
+      datasets:
       - name: validation
         type: validation
         jsonl_fpath: resources_servers/mini_swe_resource/data/validation.jsonl
@@ -31,15 +56,12 @@ mini_swe_simple_agent:
           version: 0.0.1
           artifact_fpath: validation.jsonl
         license: MIT
-      - name: example
-        type: example
-        jsonl_fpath: resources_servers/mini_swe_resource/data/example.jsonl
       concurrency: 16
       env: singularity
       cache_dir_template: ???
       run_golden: False
-      step_timeout: 600
-      eval_timeout: 1800
+      step_timeout: 300
+      eval_timeout: 900
       skip_if_exists: False
-      step_limit: 250
+      step_limit: 150
       collapse_limit: 3

resources_servers/mini_swe_resource/data/train_metrics.json

@@ -0,0 +1,97 @@
+{
+    "name": "train",
+    "type": "train",
+    "jsonl_fpath": "resources_servers/mini_swe_resource/data/train.jsonl",
+    "num_repeats": 100,
+    "gitlab_identifier": {
+        "dataset_name": "mini_swe_agent",
+        "version": "0.0.1",
+        "artifact_fpath": "train.jsonl"
+    },
+    "license": "MIT",
+    "Number of examples": 240100,
+    "Number of tools": {
+        "Total # non-null values": 0,
+        "Average": 0.0,
+        "Min": 0.0,
+        "Max": 0.0,
+        "Median": 0.0,
+        "Standard deviation": 0.0
+    },
+    "Json-dumped number of words (proxy for token count)": {
+        "Total # non-null values": 240100,
+        "Average": 2.0,
+        "Min": 2.0,
+        "Max": 2.0,
+        "Median": 2.0,
+        "Standard deviation": 0.0
+    },
+    "Number of turns": {
+        "Total # non-null values": 0,
+        "Average": 0.0,
+        "Min": 0.0,
+        "Max": 0.0,
+        "Median": 0.0,
+        "Standard deviation": 0.0
+    },
+    "Temperature": {
+        "Total # non-null values": 0,
+        "Average": 0.0,
+        "Min": 0.0,
+        "Max": 0.0,
+        "Median": 0.0,
+        "Standard deviation": 0.0
+    },
+    "instance_id": {
+        "unique_count": 2401,
+        "total_count": 240100
+    },
+    "hints_text": {
+        "unique_count": 1489,
+        "total_count": 240100
+    },
+    "patch": {
+        "unique_count": 2401,
+        "total_count": 240100
+    },
+    "test_patch": {
+        "unique_count": 2400,
+        "total_count": 240100
+    },
+    "created_at": {
+        "unique_count": 2401,
+        "total_count": 240100
+    },
+    "problem_statement": {
+        "unique_count": 2374,
+        "total_count": 240100
+    },
+    "repo": {
+        "unique_count": 11,
+        "total_count": 240100
+    },
+    "base_commit": {
+        "unique_count": 2132,
+        "total_count": 240100
+    },
+    "version": {
+        "unique_count": 160,
+        "total_count": 240100
+    },
+    "PASS_TO_PASS": {
+        "unique_count": 303293,
+        "total_count": 178030500
+    },
+    "FAIL_TO_PASS": {
+        "unique_count": 22310,
+        "total_count": 2384000
+    },
+    "subset": {
+        "unique_count": 1,
+        "total_count": 240100
+    },
+    "split": {
+        "unique_count": 1,
+        "total_count": 240100
+    }
+}

resources_servers/mini_swe_resource/data/validation_metrics.json

@@ -0,0 +1,105 @@
+{
+    "name": "validation",
+    "type": "validation",
+    "jsonl_fpath": "resources_servers/mini_swe_resource/data/validation.jsonl",
+    "num_repeats": 1,
+    "gitlab_identifier": {
+        "dataset_name": "mini_swe_agent",
+        "version": "0.0.1",
+        "artifact_fpath": "validation.jsonl"
+    },
+    "license": "MIT",
+    "Number of examples": 500,
+    "Number of tools": {
+        "Total # non-null values": 0,
+        "Average": 0.0,
+        "Min": 0.0,
+        "Max": 0.0,
+        "Median": 0.0,
+        "Standard deviation": 0.0
+    },
+    "Json-dumped number of words (proxy for token count)": {
+        "Total # non-null values": 500,
+        "Average": 2.0,
+        "Min": 2.0,
+        "Max": 2.0,
+        "Median": 2.0,
+        "Standard deviation": 0.0
+    },
+    "Number of turns": {
+        "Total # non-null values": 0,
+        "Average": 0.0,
+        "Min": 0.0,
+        "Max": 0.0,
+        "Median": 0.0,
+        "Standard deviation": 0.0
+    },
+    "Temperature": {
+        "Total # non-null values": 0,
+        "Average": 0.0,
+        "Min": 0.0,
+        "Max": 0.0,
+        "Median": 0.0,
+        "Standard deviation": 0.0
+    },
+    "repo": {
+        "unique_count": 12,
+        "total_count": 500
+    },
+    "instance_id": {
+        "unique_count": 500,
+        "total_count": 500
+    },
+    "base_commit": {
+        "unique_count": 499,
+        "total_count": 500
+    },
+    "patch": {
+        "unique_count": 500,
+        "total_count": 500
+    },
+    "test_patch": {
+        "unique_count": 500,
+        "total_count": 500
+    },
+    "problem_statement": {
+        "unique_count": 500,
+        "total_count": 500
+    },
+    "hints_text": {
+        "unique_count": 338,
+        "total_count": 500
+    },
+    "created_at": {
+        "unique_count": 500,
+        "total_count": 500
+    },
+    "version": {
+        "unique_count": 53,
+        "total_count": 500
+    },
+    "FAIL_TO_PASS": {
+        "unique_count": 494,
+        "total_count": 500
+    },
+    "PASS_TO_PASS": {
+        "unique_count": 489,
+        "total_count": 500
+    },
+    "environment_setup_commit": {
+        "unique_count": 80,
+        "total_count": 500
+    },
+    "difficulty": {
+        "unique_count": 4,
+        "total_count": 500
+    },
+    "subset": {
+        "unique_count": 1,
+        "total_count": 500
+    },
+    "split": {
+        "unique_count": 1,
+        "total_count": 500
+    }
+}