RL + Gym Rollout logs contains mostly nulls as attribute values #433
Description
Describe the bug
NeMo RL GRPO training scripts save rollout results in wandb as run.summary logs. In datasets that have multiple tools, it appears as if its saving redundant attribute information in each example. And when a tool / attribute is not triggered, it saves as "null". This makes the log very noisy and hard to interpret.
A short excerpt of the logs
"responses_create_params":{"background":null,"include":null,"input":[{"content":"Today's date is Thursday, 2023-11-30 and the current time is 23:59:00. Remember the current date and time when answering queries. Meetings must not start before 9am or end after 6pm.","role":"system","type":"message"},{"content":"Can you make a histogram of total visits on September 6?","role":"user","type":"message"}],"instructions":null,"max_output_tokens":null,"max_tool_calls":null,"metadata":null,"model":null,"parallel_tool_calls":false,"previous_response_id":null,"prompt":null,"reasoning":null,"service_tier":null,"store":null,"temperature":1.0,"text":null,"tool_choice":"auto","tools":[{"name":"company_directory_find_email_address","parameters":{"type":"object","properties":{"name":{"type":"string","description":"Name or partial name to search for in email addresses"},"email_id":null,"field":null,"query":null,"date_min":null,"date_max":null,"page":null,"page_size":null,"recipient":null,"subject":null,"body":null,"event_id":null,"time_min":null,"time_max":null,"event_name":null,"participant_email":null,"event_start":null,"duration":null,"new_value":null,"visitor_id":null,"value_to_plot":null,"plot_type":null,"traffic_source":null,"task_id":null,"task_name":null,"assigned_to_email":null,"list_name":null,"due_date":null,"board":null,"customer_name":null,"customer_email":null,"product_interest":null,"status":null,"last_contact_date_min":null,"last_contact_date_max":null,"follow_up_by_min":null,"follow_up_by_max":null,"customer_id":null,"customer_phone":null,"last_contact_date":null,"notes":null,"follow_up_by":null},"required":[],"additionalProperties":false},"strict":false,"type":"function","description":"Finds all email addresses containing the given name (case-insensitive search)."},{"name":"email_get_email_information_by_id","parameters":{"type":"object","properties":{"name":null,"email_id":{"type":"string","description":"Unique ID of the email"},"field":{"type":"string","description":"Specific field to return. Available fields: 'email_id', 'inbox/outbox', 'sender/recipient', 'subject', 'sent_datetime', 'body'"},"query":null,"date_min":null,"date_max":null,"page":null,"page_size":null,"recipient":null,"subject":null,"body":null,"event_id":null,"time_min":null,"time_max":null,"event_name":null,"participant_email":null,"event_start":null,"duration":null,"new_value":null,"visitor_id":null,"value_to_plot":null,"plot_type":null,"traffic_source":null,"task_id":null,"task_name":null,"assigned_to_email":null,"list_name":null,"due_date":null,"board":null,"customer_name":null,"customer_email":null,"product_interest":null,"status":null,"last_contact_date_min":null,"last_contact_date_max":null,"follow_up_by_min":null,"follow_up_by_max":null,"customer_id":null,"customer_phone":null,"last_contact_date":null,"notes":null,"follow_up_by":null},"required":["email_id","field"],"additionalProperties":false},"strict":false,"type":"function","description":"Retrieves specific details of an email by its ID."},{"name":"email_search_emails","parameters":{"type":"object","properties":{"name":null,"email_id":null,"field":null,"query":{"type":"string","description":"Search query, matching terms in subject, body, or sender/recipient fields"},"date_min":{"type":"string","description":"Lower date limit for the email's sent date (inclusive). Format: YYYY-MM-DD"},"date_max":{"type":"string","description":"Upper date limit for the email's sent date (inclusive). Format: YYYY-MM-DD"},"page":{"type":"integer","description":"Page number of results to return"},"page_size":{"type":"integer","description":"Number of emails per page"},"recipient":null,"subject":null,"body":null,"event_id":null,"time_min":null,"time_max":null,"event_name":null,"participant_email":null,"event_start":null,"duration":null,"new_value":null,"visitor_id":null,"value_to_plot":null,"plot_type":null,"traffic_source":null,"task_id":null,"task_name":null,"assigned_to_email":null,"list_name":null,"due_date":null,"board":null,"customer_name":null,"customer_email":null,"product_interest":null,"status":null,"last_contact_date_min":null,"last_contact_date_max":null,"follow_up_by_min":null,"follow_up_by_max":null,"customer_id":null,"customer_phone":null,"last_contact_date":null,"notes":null,"follow_up_by":null},"required":[],"additionalProperties":false},"strict":false,"type":"function","description":"Searches for emails matching the given query across subject, body, or sender fields. The function matches an email if all words in the query appear in any of these fields."},{"name":"email_send_email","parameters":{"type":"object","properties":{"name":null,"email_id":null,"field":null,"query":null,"date_min":null,"date_max":null,"page":null,"page_size":null,"recipient":{"type":"string","description":"Email address of the recipient"},"subject":{"type":"string","description":"Subject line of the email"},"body":{"type":"string","description":"Body content of the email"},"event_id":null,"time_min":null,"time_max":null,"event_name":null,"participant_email":null,"event_start":null,"duration":null,"new_value":null,"visitor_id":null,"value_to_plot":null,"plot_type":null,"traffic_source":null,"task_id":null,"task_name":null,"assigned_to_email":null,"list_name":null,"due_date":null,"board":null,"customer_name":null,"customer_email":null,"product_interest":null,"status":null,"last_contact_date_min":null,"last_contact_date_max":null,"follow_up_by_min":null,"follow_up_by_max":null,"customer_id":null,"customer_phone":null,"last_contact_date":null,"notes":null,"follow_up_by":null},"required":["recipient","subject","body"],"additionalProperties":false},"strict":false,"type":"function","description":"Sends an email to the specified recipient."},{"name":"email_delete_email","parameters":{"type":"object","properties":{"name":null,"email_id":{"type":"string","description":"Unique ID of the email to be deleted"},"field":null,"query":null,"date_min":null,"date_max":null,"page":null,"page_size":null,"recipient":null,"subject":null,"body":null,"event_id":null,"time_min":null,"time_max":null,"event_name":null,"participant_email":null,"event_start":null,"duration":null,"new_value":null,"visitor_id":null,"value_to_plot":null,"plot_type":null,"traffic_source":null,"task_id":null,"task_name":null,"assigned_to_email":null,"list_name":null,"due_date":null,"board":null,"customer_name":null,"customer_email":null,"product_interest":null,"status":null,"last_contact_date_min":null,"last_contact_date_max":null,"follow_up_by_min":null,"follow_up_by_max":null,"customer_id":null,"customer_phone":null,"last_contact_date":null,"notes":null,"follow_up_by":null},"required":["email_id"],"additionalProperties":false},"strict":false,"type":"function","description":"Deletes an email by its ID."},{"name":"email_forward_email","parameters":{"type":"object","properties":{"name":null,"email_id":{"type":"string","description":"Unique ID of the email to be forwarded"},"field":null,"query":null,"date_min":null,"date_max":null,"page":null,"page_size":null,"recipient":{"type":"string","description":"Email address of the recipient"},"subject":null,"body":null,"event_id":null,"time_min":null,"time_max":null,"event_name":null,"participant_email":null,"event_start":null,"duration":null,"new_value":null,"visitor_id":null,"value_to_plot":null,"plot_type":null,"traffic_source":null,"task_id":null,"task_name":null,"assigned_to_email":null,"list_name":null,"due_date":null,"board":null,"customer_name":null,"customer_email":null,"product_interest":null,"status":null,"last_contact_date_min":null,"last_contact_date_max":null,"follow_up_by_min":null,"follow_up_by_max":null,"customer_id":null,"customer_phone":null,"last_contact_date":null,"notes":null,"follow_up_by":null},"required":["email_id","recipient"],"additionalProperties":false},"strict":false,"type":"function","description":"Forwards an email to the specified recipient."},{"name":"email_reply_email","parameters":{"type":"object","properties":{"name":null,"email_id":{"type":"string","description":"Unique ID of the email to be replied"},"field":null,"query":null,"date_min":null,"date_max":null,"page":null,"page_size":null,"recipient":null,"subject":null,"body":{"type":"string","description":"Body content of the email"},"event_id":null,"time_min":null,"time_max":null,"event_name":null,"participant_email":null,"event_start":null,"duration":null,"new_value":null,"visitor_id":null,"value_to_plot":null,"plot_type":null,"traffic_source":null,"task_id":null,"task_name":null,"assigned_to_email":null,"list_name":null,"due_date":null,"board":null,"customer_name":null,"customer_email":null,"product_interest":null,"status":null,"last_contact_date_min":null,"last_contact_date_max":null,"follow_up_by_min":null,"follow_up_by_max":null,"customer_id":null,"customer_phone":null,"last_contact_date":null,"notes":null,"follow_up_by":null},"required":["email_id","body"],"additionalProperties":false},"strict":false,"type":"function","description":"Replies to an email by its ID."},{"name":"calendar_get_event_information_by_id","parameters":{"type":"object","properties":{"name":null,"email_id":null,"field":{"type":"string","description":"Field to return. Available fields are: 'event_id', 'event_name', 'participant_email', 'event_start', 'duration'"},"query":null,"date_min":null,"date_max":null,"page":null,"page_size":null,"recipient":null,"subject":null,"body":null,"event_id":{"type":"string","description":"8-digit ID of the event"},"time_min":null,"time_max":null,"event_name":null,"participant_email":null,"event_start":null,"duration":null,"new_value":null,"visitor_id":null,"value_to_plot":null,"plot_type":null,"traffic_source":null,"task_id":null,"task_name":null,"assigned_to_email":null,"list_name":null,"due_date":null,"board":null,"customer_name":null,"customer_email":null,"product_interest":null,"status":null,"last_contact_date_min":null,"last_contact_date_max":null,"follow_up_by_min":null,"follow_up_by_max":null,"customer_id":null,"customer_phone":null,"last_contact_date":null,"notes":null,"follow_up_by":null},"required":["event_id","field"],"additionalProperties":false},"strict":false,"type":"function","description":"Returns the event for a given ID."},{"name":"calendar_search_events","parameters":{"type":"object","properties":{"name":null,"email_id":null,"field":null,"query":{"type":"string","description":"Query to search for. Terms will be matched in the event_name and participant_email fields"},"date_min":null,"date_max":null,"page":{"type":"integer","description":"Page number of results to return"},"page_size":{"type":"integer","description":"Number of events per page"},"recipient":null,"subject":null,"body":null,"event_id":null,"time_min":{"type":"string","description":"Lower bound (inclusive) for an event's end time to filter by. Format: YYYY-MM-DD HH:MM:SS"},"time_max":{"type":"string","description":"Upper bound (inclusive) for an event's start time to filter by. Format: YYYY-MM-DD HH:MM:SS"},"event_name":null,"participant_email":null,"event_start":null,"duration":null,"new_value":null,"visitor_id":null,"value_to_plot":null,"plot_type":null,"traffic_source":null,"task_id":null,"task_name":null,"assigned_to_email":null,"list_name":null,"due_date":null,"board":null,"customer_name":null,"customer_email":null,"product_interest":null,"status":null,"last_contact_date_min":null,"last_contact_date_max":null,"follow_up_by_min":null,"follow_up_by_max":null,"customer_id":null,"customer_phone":null,"last_contact_date":null,"notes":null,"follow_up_by":null},"required":[],"additionalProperties":false},"strict":false,"type":"function","description":"Returns the events for a given query with pagination support."},{"name":"calendar_create_event","parameters":{"type":"object","properties":{"name":null,"email_id":null,"field":null,"query":null,"date_min":null,"date_max":null,"page":null,"page_size":null,"recipient":null,"subject":null,"body":null,"event_id":null,"time_min":null,"time_max":null,"event_name":{"type":"string","description":"Name of the event"},"participant_email":{"type":"string","description":"Email of the participant"},"event_start":{"type":"string","description":"Start time of the event. Format: YYYY-MM-DD HH:MM:SS"},"duration":{"type":"string","description":"Duration of the event in minutes"},"new_value":null,"visitor_id":null,"value_to_plot":null,"plot_type":null,"traffic_source":null,"task_id":null,"task_name":null,"assigned_to_email":null,"list_name":null,"due_date":null,"board":null,"customer_name":null,"customer_email":null,"product_interest":null,"status":null,"last_contact_date_min":null,"last_contact_date_max":null,"follow_up_by_min":null,"follow_up_by_max":null,"customer_id":null,"customer_phone":null,"last_contact_date":null,"notes":null,"follow_up_by":null},"required":["event_name","participant_email","event_start","duration"],"additionalProperties":false},"strict":false,"type":"function","description":"Creates a new event."},{"name":"calendar_delete_event","parameters":{"type":"object","properties":{"name":null,"email_id":null,"field":null,"query":null,"date_min":null,"date_max":null,"page":null,"page_size":null,"recipient":null,"subject":null,"body":null,"event_id":{"type":"string","description":"8-digit ID of the event"},"time_min":null,"time_max":null,"event_name":null,"participant_email":null,"event_start":null,"duration":null,"new_value":null,"visitor_id":null,"value_to_plot":null,"plot_type":null,"traffic_source":null,"task_id":null,"task_name":null,"assigned_to_email":null,"list_name":null,"due_date":null,"board":null,"customer_name":null,"customer_email":null,"product_interest":null,"status":null,"last_contact_date_min":null,"last_contact_date_max":null,"follow_up_by_min":null,"follow_up_by_max":null,"customer_id":null,"customer_phone":null,"last_contact_date":null,"notes":null,"follow_up_by":null},"required":["event_id"],"additionalProperties":false},"strict":false,"type":"function","description":"Deletes an event."},{"name":"calendar_update_event","parameters":
Steps/Code to reproduce bug
Run GRPO with the workplace_assistant environment.
I have branches that add a tutorial that uses Nemotron nano v2 9B model with this environment.
NeMo-RL branch: https://github.com/shashank3959/RL/tree/dev/shashankv-add-grpo-workplace-asst
NeMo-Gym branch: https://github.com/shashank3959/gym/tree/dev/shashankv-add-grpo-workplace-asst
Expected behavior
Logs are clear enough to debug any issues.
Configs
NeMo Gym (e.g. via ng_dump_config) or RL training framework config files.
Environment details
Additional context
Running on a SLURM cluster with a 8xH100 node.
Metadata
Metadata
Assignees
Labels
No labels