authoravatarclover caruso<git@paperclover.net>2026-10-12 23:30:06-07:00
committeravatarclover caruso<git@paperclover.net>2026-10-21 02:52:29-07:00
log843ea131e1e67ca7bfc5d0cc838041469099038d
tree0b0a1b3d1b25d4d05cf6f7ca01dbdb5ccb051c92
parenta2f8bb12305846efcb74fa7657ccbcc07c7aceb7
signature Signed by SSH key SHA256:52mNGHRsVFBDED9IAX5pe+LRWUefqTbxEReunq21QvU

Upgrade Shale to r1807 and read issues as JSON

Negotiate JSON on existing repository and issue URLs while preserving broker permissions and model attribution. Retain HTML for account identity and label IDs, and disable preview mirror side effects. Verified native OIDC and MCP writes, revocation, restart persistence, private access, concurrent JSON and Markdown reads, and unchanged preview data. Assisted-by: gpt-6

5 files changed, 91 insertions(+), 240 deletions(-)

dashboard/src/shale.rs+69-237
......@@ -12,19 +12,6 @@ use scraper::{Html, Selector};
1212
1313const MODEL_PATTERN: &str = r"^[a-z0-9][a-z0-9._-]{0,127}$";
1414
15fn tokenless_r1616(document: &Html) -> bool {
16 if document
17 .select(&Selector::parse("input[name=csrf_token]").unwrap())
18 .next()
19 .is_some()
20 {
21 return false;
22 }
23 let links: Vec<_> = document.select(&Selector::parse("body#page-issue > footer.usa-footer .usa-footer__secondary-section a[href='https://astheno.software/shale/']").unwrap()).collect();
24 matches!(links.as_slice(), [link] if text(*link) == "shale r1616-ga87d2f5.zig.0.16.0")
25}
26
27
2815pub struct Backend {
2916 pub(crate) origin: url::Url,
3017 http: reqwest::Client,
......@@ -57,15 +44,14 @@ impl Backend {
5744 path: &str,
5845 session: Option<&str>,
5946 ) -> Result<(StatusCode, HeaderMap, String)> {
60 self.request(Method::GET, self.origin.join(path)?, session, None)
47 self.request(self.origin.join(path)?, session, "text/html")
6148 .await
6249 }
6350 async fn request(
6451 &self,
65 method: Method,
6652 target: url::Url,
6753 session: Option<&str>,
68 form: Option<&HashMap<String, String>>,
54 accept: &str,
6955 ) -> Result<(StatusCode, HeaderMap, String)> {
7056 if target.origin() != self.origin.origin()
7157 || !target.username().is_empty()
......@@ -77,18 +63,12 @@ impl Backend {
7763 let _slot = tokio::time::timeout(Duration::from_secs(5), self.slots.acquire())
7864 .await
7965 .map_err(|_| Error::new(429, "Shale is busy. Try again in a moment."))??;
80 let mut request = self.http.request(method, target.clone());
66 let mut request = self.http.get(target.clone()).header("accept", accept);
8167 if let Some(session) = session {
8268 let mut cookie = axum::http::HeaderValue::from_str(&format!("SessionID={session}"))?;
8369 cookie.set_sensitive(true);
8470 request = request.header("cookie", cookie);
8571 }
86 if let Some(form) = form {
87 request = request
88 .header("origin", self.origin.origin().ascii_serialization())
89 .header("referer", target.as_str())
90 .form(form);
91 }
9272 let mut response = request
9373 .send()
9474 .await
......@@ -112,9 +92,9 @@ impl Backend {
11292 }
11393 Ok((status, headers, String::from_utf8(body)?))
11494 }
115 async fn page(&self, target: &url::Url, session: &str) -> Result<String> {
95 async fn page(&self, target: &url::Url, session: &str, accept: &str) -> Result<String> {
11696 let (status, _, body) = self
117 .request(Method::GET, target.clone(), Some(session), None)
97 .request(target.clone(), Some(session), accept)
11898 .await?;
11999 match status {
120100 StatusCode::OK => Ok(body),
......@@ -135,30 +115,13 @@ impl Backend {
135115 )),
136116 }
137117 }
138}
139
140fn document(html: &str, page: &str, repository: Option<&str>) -> Result<Html> {
141 let document = Html::parse_document(html);
142 if document
143 .select(&Selector::parse("body").unwrap())
144 .next()
145 .and_then(|body| body.attr("id"))
146 != Some(page)
147 || repository.is_some_and(|name| {
148 document
149 .select(&Selector::parse("meta[name='astheno.shale.repo.name']").unwrap())
150 .next()
151 .and_then(|meta| meta.attr("content"))
152 != Some(name)
153 })
154 {
155 return Err(Error::new(
156 502,
157 "Shale's page changed. Open it to check the result.",
158 ));
118 async fn json(&self, target: &url::Url, session: &str) -> Result<Value> {
119 let body = self.page(target, session, "application/json").await?;
120 serde_json::from_str(&body)
121 .map_err(|_| Error::new(502, "Shale didn't return JSON. Open it to check the result."))
159122 }
160 Ok(document)
161123}
124
162125fn text(element: scraper::ElementRef<'_>) -> String {
163126 element.text().collect::<String>().trim().to_owned()
164127}
......@@ -206,137 +169,52 @@ pub(crate) async fn verified_session(app: &App, owner: &str) -> Result<String> {
206169 let session = session(app, owner)?;
207170 username(
208171 &app.shale
209 .page(&app.shale.origin.join("/-/settings")?, &session)
172 .page(&app.shale.origin.join("/-/settings")?, &session, "text/html")
210173 .await?,
211174 )?;
212175 Ok(session)
213176}
214177pub async fn repositories(app: &App, owner: &str) -> Result<Vec<Value>> {
215178 let session = verified_session(app, owner).await?;
216 let body = app.shale.page(&app.shale.origin, &session).await?;
217 let document = document(&body, "page-index", None)?;
179 let value = app.shale.json(&app.shale.origin, &session).await?;
180 let entries = value.as_array().filter(|v| v.len() <= 1024)
181 .ok_or_else(|| Error::new(502, "Shale's repository list changed. Open Shale to browse it."))?;
218182 let mut repositories = Vec::new();
219 for row in document.select(&Selector::parse(".grid-container table tbody tr").unwrap()) {
220 let cells: Vec<_> = row.select(&Selector::parse("td").unwrap()).collect();
221 if cells.len() != 3 {
222 return Err(Error::new(
223 502,
224 "Shale's repository list changed. Open Shale to browse it.",
225 ));
226 }
227 let link = cells[0]
228 .select(&Selector::parse("a").unwrap())
229 .next()
230 .ok_or_else(|| {
231 Error::new(
232 502,
233 "Shale's repository list changed. Open Shale to browse it.",
234 )
235 })?;
236 let name = text(link);
237 let target = app
238 .shale
239 .origin
240 .join(link.attr("href").unwrap_or_default())?;
241 if target != repository_path(&app.shale.origin, &name, &[""])?
242 || repositories.iter().any(|r: &Value| r["id"] == name)
243 {
244 return Err(Error::new(
245 502,
246 "Shale's repository list changed. Open Shale to browse it.",
247 ));
183 for entry in entries {
184 let name = entry["name"].as_str()
185 .ok_or_else(|| Error::new(502, "Shale's repository list changed. Open Shale to browse it."))?;
186 repository_path(&app.shale.origin, name, &[""])?;
187 if repositories.iter().any(|r: &Value| r["id"] == name) {
188 return Err(Error::new(502, "Shale's repository list changed. Open Shale to browse it."));
248189 }
249 repositories.push(json!({"id":name,"name":name,"description":text(cells[1])}));
250 }
251 if repositories.len() > 1024 {
252 return Err(Error::new(
253 502,
254 "The repository list is too large. Open Shale to narrow it.",
255 ));
190 repositories.push(json!({"id":name,"name":name,"description":entry["description"]}));
256191 }
257192 Ok(repositories)
258193}
259fn issue(html: &str, repository: &str, id: Option<u64>) -> Result<Value> {
260 let document = document(html, "page-issue", Some(repository))?;
261 let spans: Vec<_> = document
262 .select(&Selector::parse("h1 > span").unwrap())
263 .collect();
264 let (issue_id, title) = if spans.len() == 2 {
265 (
266 text(spans[0])
267 .strip_prefix('#')
268 .and_then(|id| id.parse::<u64>().ok()),
269 Some(text(spans[1])),
270 )
271 } else if spans.is_empty() && tokenless_r1616(&document) {
272 let headings: Vec<_> = document.select(&Selector::parse("h1").unwrap()).collect();
273 if let [heading] = headings.as_slice() {
274 text(*heading)
275 .strip_prefix("Issue #")
276 .and_then(|text| text.split_once(": "))
277 .map(|(id, title)| (id.parse::<u64>().ok(), Some(title.to_owned())))
278 .unwrap_or((None, None))
279 } else {
280 (None, None)
281 }
282 } else {
283 (None, None)
284 };
285 let statuses: Vec<_> = document
286 .select(&Selector::parse("dl.sidebar dd span[class*='issuestatus-']").unwrap())
287 .filter_map(|status| {
288 status
289 .value()
290 .classes()
291 .find_map(|class| class.strip_prefix("issuestatus-"))
292 })
293 .collect();
294 let status = match statuses.as_slice() {
295 [status] => Some(*status),
296 _ => None,
297 };
298 if title.as_ref().is_none_or(|title| title.is_empty())
299 || issue_id.is_none_or(|n| n == 0 || id.is_some_and(|id| n != id))
300 || status.is_none()
194fn issue(value: Value, repository: &str, id: u64) -> Result<Value> {
195 let issue = &value["issue"];
196 if id == 0 || issue["number"].as_u64() != Some(id)
197 || !issue["title"].as_str().is_some_and(|s| !s.is_empty())
198 || !issue["status"].as_str().is_some_and(|s| !s.is_empty())
199 || !value["actions"].is_array() || !value["labels"].is_array()
301200 {
302 return Err(Error::new(
303 502,
304 "Shale's issue page changed. Open the issue to check it.",
305 ));
306 }
307 let mut comments = Vec::new();
308 for comment in document.select(&Selector::parse("li.comment").unwrap()) {
309 let content = comment
310 .select(&Selector::parse(".markdown").unwrap())
311 .next()
312 .ok_or_else(|| {
313 Error::new(
314 502,
315 "Shale's comments changed. Open the issue to read them.",
316 )
317 })?;
318 let author = comment
319 .select(&Selector::parse(".n-card__header span.usa-link.va-middle-childs, .n-card__header a[href^='/~'], .n-card__header a[href^='https://github.com/'], .n-card__header a:not([href]).va-middle-childs").unwrap())
320 .next()
321 .map(text);
322 let time = comment
323 .select(&Selector::parse(".n-card__header span[title]").unwrap())
324 .next()
325 .and_then(|e| e.attr("title"));
326 comments.push(
327 json!({"id":comment.attr("id"),"author":author,"createdAt":time,"text":text(content)}),
328 );
201 return Err(Error::new(502, "Shale's issue response changed. Open the issue to check it."));
329202 }
330 let labels: Vec<_> = document
331 .select(&Selector::parse("dd.sidebar-labels a").unwrap())
332 .map(text)
203 let comments: Vec<_> = array(&value["actions"]).iter()
204 .filter(|action| action["kind"] == "comment" && action["deleted_on"].is_null())
205 .map(|action| json!({"author":action["actor"],"createdAt":action["added_on"],"text":action["payload"]}))
333206 .collect();
334 Ok(
335 json!({"repository":repository,"id":issue_id,"title":title.unwrap(),"status":status,"labels":labels,"comments":comments}),
336 )
207 Ok(json!({"repository":repository,"id":id,"title":issue["title"],"status":issue["status"],"labels":value["labels"],"comments":comments}))
337208}
209
338210fn labels(html: &str, repository: &str) -> Result<Value> {
339 let document = document(html, "page-labels", Some(repository))?;
211 let document = Html::parse_document(html);
212 if document.select(&Selector::parse("body#page-labels").unwrap()).next().is_none()
213 || document.select(&Selector::parse("meta[name='astheno.shale.repo.name']").unwrap()).next()
214 .and_then(|meta| meta.attr("content")) != Some(repository)
215 {
216 return Err(Error::new(502, "Shale's label page changed. Open its labels page."));
217 }
340218 let mut labels = Vec::new();
341219 for row in document.select(&Selector::parse(".grid-container table tbody tr").unwrap()) {
342220 let cells: Vec<_> = row.select(&Selector::parse("td").unwrap()).collect();
......@@ -477,7 +355,7 @@ impl ServerHandler for Shale {
477355 }
478356 if name == "list_labels" {
479357 let target = repository_path(&app.shale.origin, repository, &["issues", "labels"])?;
480 let result = labels(&app.shale.page(&target, &credential).await?, repository)?;
358 let result = labels(&app.shale.page(&target, &credential, "text/html").await?, repository)?;
481359 current(app, grant, &credential)?;
482360 return Ok(result);
483361 }
......@@ -487,7 +365,6 @@ impl ServerHandler for Shale {
487365 .ok_or_else(|| Error::new(400, "Choose a positive issue ID."))?)
488366 };
489367 if let Some(id) = issue_id { target = repository_path(&app.shale.origin, repository, &["issues", &id.to_string()])?; }
490 if name == "create_issue" { target = repository_path(&app.shale.origin, repository, &["issues", "new"])?; }
491368 if let Some(q) = arguments.get("q") {
492369 let q = q.as_str().filter(|q| q.len() <= 4096).ok_or_else(|| Error::new(400, "Narrow the issue search."))?;
493370 target.query_pairs_mut().append_pair("q", q);
......@@ -514,34 +391,26 @@ impl ServerHandler for Shale {
514391 if body.len() > 262144 { return Err(Error::new(400, "Shorten the comment to leave room for its Assisted-by trailer.")); }
515392 }
516393 }
517 username(&app.shale.page(&app.shale.origin.join("/-/settings")?, &credential).await?)?;
518 let body = app.shale.page(&target, &credential).await?;
394 username(&app.shale.page(&app.shale.origin.join("/-/settings")?, &credential, "text/html").await?)?;
519395 if name == "list_issues" {
520 let document = document(&body, "page-issues", Some(repository))?;
396 let value = app.shale.json(&target, &credential).await?;
397 let rows = value.as_array()
398 .ok_or_else(|| Error::new(502, "Shale's issue list changed. Open it in Shale."))?;
521399 let mut issues = Vec::new();
522 for row in document.select(&Selector::parse(".grid-container table tbody tr").unwrap()) {
523 let cells: Vec<_> = row.select(&Selector::parse("td").unwrap()).collect();
524 if cells.len() != 6 { return Err(Error::new(502, "Shale's issue list changed. Open it in Shale.")); }
525 let id = text(cells[0]).strip_prefix('#').and_then(|s| s.parse::<u64>().ok()).filter(|n| *n > 0)
400 for row in rows {
401 let id = row["number"].as_u64().filter(|id| *id > 0)
526402 .ok_or_else(|| Error::new(502, "Shale's issue list changed. Open it in Shale."))?;
527 let status = cells[1].select(&Selector::parse("span[class]").unwrap())
528 .flat_map(|span| span.value().classes()).find_map(|class| class.strip_prefix("issuestatus-"))
529 .filter(|status| !status.is_empty())
530 .ok_or_else(|| Error::new(502, "Shale's issue list changed. Open it in Shale."))?;
531 issues.push(json!({"id":id,"title":text(cells[2]),"status":status,"author":text(cells[3]),
532 "modifiedAt":cells[4].select(&Selector::parse("span[title]").unwrap()).next().and_then(|e| e.attr("title")),
533 "createdAt":cells[5].select(&Selector::parse("span[title]").unwrap()).next().and_then(|e| e.attr("title"))}));
403 issues.push(json!({"id":id,"title":row["title"],"status":row["status"],"author":row["author"],"modifiedAt":row["last_updated"]}));
534404 }
535405 current(app, grant, &credential)?;
536406 return Ok(json!({"repository":repository,"issues":issues}));
537407 }
538408 if !write {
539 let issue = issue(&body, repository, issue_id)?;
409 let issue = issue(app.shale.json(&target, &credential).await?, repository, issue_id.unwrap())?;
540410 current(app, grant, &credential)?;
541411 return Ok(json!({"issue":issue}));
542412 }
543413 if name == "create_issue" {
544 document(&body, "page-new-issue", Some(repository))?;
545414 let status = match arguments.get("status").and_then(Value::as_str) {
546415 None if !arguments.contains_key("status") => "needs_triage",
547416 Some("triage") => "needs_triage",
......@@ -549,7 +418,9 @@ impl ServerHandler for Shale {
549418 _ => return Err(Error::new(400, "Choose todo or triage for a new issue.")),
550419 };
551420 fields.insert("status".into(), status.into());
552 } else { issue(&body, repository, issue_id)?; }
421 } else {
422 issue(app.shale.json(&target, &credential).await?, repository, issue_id.unwrap())?;
423 }
553424 let mut fields = serde_json::to_value(fields)?;
554425 if let Some(id) = issue_id { fields["id"] = json!(id); }
555426 fields["model"] = json!(model.unwrap());
......@@ -558,7 +429,7 @@ impl ServerHandler for Shale {
558429 .map_err(|error| if error.status >= 500 { Error::new(502, "The write outcome is unknown. Check Shale before retrying.") } else { error })?;
559430 let id = result["id"].as_u64().ok_or_else(|| Error::new(502, "The write outcome is unknown. Check Shale before retrying."))?;
560431 let target = repository_path(&app.shale.origin, repository, &["issues", &id.to_string()])?;
561 let result = issue(&app.shale.page(&target, &credential).await?, repository, Some(id))?;
432 let result = issue(app.shale.json(&target, &credential).await?, repository, id)?;
562433 current(app, grant, &credential).map_err(|_| Error::new(502, "The write outcome is unknown. Check Shale before retrying."))?;
563434 Ok(json!({"issue":result}))
564435 }.await;
......@@ -923,62 +794,23 @@ mod tests {
923794 );
924795 }
925796 #[test]
926 fn issue_pages_must_match_repository_identity_and_issue_number() {
927 let page = "<meta name='astheno.shale.repo.name' content='owned'><body id=page-issue><h1><span>#3</span><span>Snow &amp; ☃</span></h1><dl class=sidebar><dd><span class=issuestatus-done><span>Done</span></span></dd></dl>";
928 assert_eq!(issue(page, "owned", Some(3)).unwrap()["title"], "Snow & ☃");
929 assert_eq!(issue(page, "owned", Some(3)).unwrap()["status"], "done");
930 let owner = format!(
931 "{page}<form><select name=status><option selected value=todo>Todo</option></select></form>"
932 );
933 assert_eq!(issue(&owner, "owned", Some(3)).unwrap()["status"], "done");
934 assert!(issue(page, "other", Some(3)).is_err());
935 assert!(issue(page, "owned", Some(4)).is_err());
936 assert!(issue(&page.replace("page-issue", "page-login"), "owned", Some(3)).is_err());
937 assert!(
938 issue(
939 &page.replace("issuestatus-done", "unknown"),
940 "owned",
941 Some(3)
942 )
943 .is_err()
944 );
945 assert!(
946 issue(
947 &format!(
948 "{page}<dl class=sidebar><dd><span class=issuestatus-todo></span></dd></dl>"
949 ),
950 "owned",
951 Some(3)
952 )
953 .is_err()
954 );
955 }
956 #[test]
957 fn verified_r1616_issue_heading_keeps_identity_checks() {
958 let page = "<head><meta name='astheno.shale.repo.name' content='owned'></head><body id=page-issue><h1>Issue #3: Snow &amp; ☃</h1><dl class=sidebar><dd><span class=issuestatus-done><span>Done</span></span></dd></dl><footer class=usa-footer><div class=usa-footer__secondary-section><a href='https://astheno.software/shale/'>shale r1616-ga87d2f5.zig.0.16.0</a></div></footer></body>";
959 assert_eq!(issue(page, "owned", Some(3)).unwrap()["title"], "Snow & ☃");
960 assert!(issue(page, "other", Some(3)).is_err());
961 assert!(issue(page, "owned", Some(4)).is_err());
962 assert!(
963 issue(
964 &page.replace("r1616-ga87d2f5.zig.0.16.0", "r1758-new"),
965 "owned",
966 Some(3)
967 )
968 .is_err()
969 );
970 assert!(issue(&page.replace("Issue #3", "Issue #0"), "owned", None).is_err());
971 }
972 #[test]
973 fn issue_comment_authors_include_external_guest_profiles() {
974 let page = "<head><meta name='astheno.shale.repo.name' content='owned'></head><body id=page-issue><h1><span>#3</span><span>Title</span></h1><dl class=sidebar><dd><span class=issuestatus-done>Done</span></dd></dl><ul><li class=comment id=c1><div class=n-card__header><a href='https://github.com/paperclover'>paperclover</a></div><div class=markdown>Comment</div></li><li class=comment id=c2><div class=n-card__header><a class='usa-link va-middle-childs'>Astheno user</a></div><div class=markdown>Comment</div></li></ul></body>";
975 let page = page.replace("</ul>", "<li class=comment id=c3><div class=n-card__header><span class='usa-link va-middle-childs'><span>GPT-6.1 Sol</span> <small>(on behalf of <a class='usa-link va-middle-childs' href='/~clover'>~clover</a>)</small></span></div><div class=markdown>Comment</div></li></ul>");
976 let parsed = issue(&page, "owned", Some(3)).unwrap();
977 assert_eq!(parsed["comments"][2]["author"], "GPT-6.1 Sol (on behalf of ~clover)");
978 assert_eq!(parsed["comments"][0]["author"], "paperclover");
979 assert_eq!(parsed["comments"][1]["author"], "Astheno user");
797 fn issue_json_keeps_markdown_and_excludes_deleted_actions() {
798 let value = json!({"issue":{"number":3,"title":"Snow & ☃","status":"done"},"labels":["bug"],"actions":[
799 {"actor":"owner","kind":"comment","payload":"**Markdown** <angle> ☃","added_on":"2026-10-13T00:00:00Z","deleted_on":null},
800 {"actor":"owner","kind":"update_status","payload":"done","deleted_on":null},
801 {"actor":"owner","kind":"comment","payload":"deleted","deleted_on":"2026-10-13T01:00:00Z"}
802 ]});
803 let parsed = issue(value.clone(), "owned", 3).unwrap();
804 assert_eq!(parsed["comments"], json!([{"author":"owner","createdAt":"2026-10-13T00:00:00Z","text":"**Markdown** <angle> ☃"}]));
805 assert_eq!(parsed["labels"], json!(["bug"]));
806 assert!(issue(value.clone(), "owned", 4).is_err());
807 assert!(issue(value.clone(), "owned", 0).is_err());
808 for field in ["issue", "actions", "labels"] {
809 let mut incomplete = value.clone();
810 incomplete.as_object_mut().unwrap().remove(field);
811 assert!(issue(incomplete, "owned", 3).is_err());
812 }
980813 }
981
982814 #[test]
983815 fn account_identity_uses_html_text_and_rejects_login_or_changed_markup() {
984816 assert_eq!(
service/shale/service.pkl+6-1
......@@ -1,6 +1,7 @@
11amends "../../config/Service.pkl"
22
33import "../../config/OpenID.pkl" as sso
4import "../../config/site.pkl" as site
45
56meta {
67 name = "Shale"
......@@ -18,7 +19,7 @@ requirements {
1819prepare = "prepare.py"
1920
2021container {
21 image = "docker.io/astheno/shale@sha256:547a126d829c81be5d2435223a093337951e2c6ae71a114adf69938485fb3fa8"
22 image = "docker.io/astheno/shale@sha256:b55cc42c4eb2c1e21733405bec795e49ddd7c71fb1755d2aa2ee4361f750cebd"
2223 extraHosts { "\(sso.hostname):host-gateway" }
2324
2425 http {
......@@ -54,6 +55,10 @@ container {
5455 ["SERVER_TITLE"] = "clover's git"
5556 // Concurrent Markdown rendering shares capture state between request workers.
5657 ["NPROC"] = "1"
58 when (site.preview) {
59 ["MIRRORS_DISABLE_CLONE"] = "true"
60 ["MIRRORS_DISABLE_PULL_POLL"] = "true"
61 }
5762 ["SESSION_SECRET"] = "${secret.own.session_secret}"
5863 ["OAUTH2_CLIENT"] = "oidc,\(sso.hostname)|${secret.oidc.clientId}|${secret.oidc.clientSecret}"
5964 }
tools/dashboard-shale-link-test.py+1-1
......@@ -354,7 +354,7 @@ def main():
354354 created = call(read, 'get_issue', {'repository': 'alpha', 'id': 1})['issue']
355355 assert created['title'] == title and created['id'] == 1
356356 assert len(created['comments']) == 1 and 'Unicode ☃' in created['comments'][0]['text']
357 assert '<script>' not in json.dumps(created)
357 assert '<script>fixture</script>' in created['comments'][0]['text']
358358 listed = call(read, 'list_issues', {'repository': 'alpha', 'q': 'is:open'})['issues'][0]
359359 assert listed['title'] == title and listed['status'] == created['status']
360360 rejected = call(read, 'list_issues', {'repository': 'alpha', 'q': 'Unicode'}, error=True)
tools/dashboard-shale-page-test.py+6
......@@ -55,6 +55,8 @@ def main():
5555 observations.append((self.command, self.path, {key.lower(): value for key, value in self.headers.items()}))
5656 if self.path == '/binary':
5757 return self.respond(blob, 'application/octet-stream')
58 if self.path == '/json':
59 return self.respond(b'{"issue":{"title":"<angle> & Unicode"}}', 'application/json; charset=utf-8')
5860 if self.path == '/large':
5961 return self.respond(b' ' * (9 * 1024 * 1024), 'text/html')
6062 if self.path == '/-/login':
......@@ -199,6 +201,10 @@ def main():
199201 assert observations[-1][1] == '/snowbound/issues/26?query=kept', observations[-1]
200202 assert observations[-1][2]['host'] == 'shale.studio.test', observations[-1]
201203 assert observations[-1][2]['cookie'] == 'SessionID=preserved', observations[-1]
204 status, headers, body = request('/json', headers={'Accept': 'application/json'})
205 assert status == 200 and headers['Content-Type'].startswith('application/json')
206 assert body == b'{"issue":{"title":"<angle> & Unicode"}}'
207 assert observations[-1][2]['accept'] == 'application/json'
202208 assert not any(key.lower().startswith('studio-') for key in observations[-1][2]), observations[-1]
203209 status, headers, body = request('/~guest-github-24465214')
204210 assert status == 302 and headers['Location'] == 'https://github.com/paperclover' and headers['Referrer-Policy'] == 'no-referrer'
tools/shale-migration.md+9-1
......@@ -52,7 +52,7 @@ The writable ZFS rehearsal `shale-preview-0242cee6` starts from all 104 existing
5252
5353Production issue import completed on October 5 under release `dda941e618934ad9`, from committed main `51be72a9`. Recovery snapshot: `globe/prod/shale@before-forgejo-issues-20261005T080143Z-14c71a`. The private import report and SQLite backup live at `/var/lib/studio/forgejo-issue-migration/issue-import-c726euv2`. All 104 native issues survived alongside the 452 imported issues. After the route reload settled, 904 concurrent HTTPS issue reads passed (644 successful reads, 260 expected private-page denials), all 24 attachment hashes and access checks passed, SQLite integrity passed, and a forged-Origin request returned 403. Browser navigation confirmed historical timestamps, comments, labels, and status on a live imported issue. Native `chat` #1 remains Todo. The disposable Keycloak rehearsal was destroyed after verification; the production recovery snapshot and private evidence remain.
5454
55This older build predates hidden form CSRF tokens. The router requires the exact HTTPS Origin for every Shale request using the `SessionID` cookie and a mutating method. The guard precedes all Shale handlers inside an explicit Caddy `route`; otherwise default directive ordering can bypass it. Missing, wrong, and suffix-forged origins returned 403 without a database change on the clone. The valid site origin allowed a status change, while Basic-auth Git requests still reached Shale. The MCP adapter permits tokenless issue forms only with the exact verified `r1616` structural footer and no CSRF-token input anywhere on the page. Newer or mixed-token markup remains strict.
55The r1616 build predated hidden form CSRF tokens. The router's exact HTTPS Origin guard preceded all Shale handlers inside an explicit Caddy `route`; default directive ordering could otherwise bypass it. Missing, wrong, and suffix-forged origins returned 403 without a database change on the clone. The valid site origin allowed a status change, while Basic-auth Git requests still reached Shale. At that time, the MCP adapter accepted tokenless issue forms only with the verified `r1616` structural footer and no CSRF-token input anywhere on the page.
5656
5757The October 5 `r1763-g9f5b1c7.zig.0.16.0` upgrade rehearsal passed anonymous reads of the formerly crashing issues, Unicode issue creation and comments, issue closing with a comment Delete form present, access denial, restart persistence, token revocation, and logout. The Rust adapter parsed its actual owner issue markup and accepted its status/comment CSRF fields. All migrated SQLite rows remained identical. Eight-worker concurrent Markdown rendering still crashed; keep `NPROC=1`, which passed 280 concurrent fenced-code requests. The injected issue-form script is no longer needed. Private evidence lives at `/var/lib/studio/shale-upgrade-0680d28c`; its disposable containers were removed after testing.
5858
......@@ -73,3 +73,11 @@ After a same-machine OS replacement, set `STUDIO_LEGACY_HANDOFF` to the [offline
7373The Forgejo importer originally omitted `http.receivepack=true`, which Shale sets when it creates an owned repository. Shale checks its push ACL before invoking `git-http-backend`, but does not set `REMOTE_USER`; Git's default therefore rejected authorized pushes to imported repositories. The importer now writes the setting, and Shale's prepare step repairs missing settings without replacing an explicit disable. Production `config` remains owner-only for pushes.
7474
7575Shale also forwards CGI `Status` as an ordinary header on HTTP 200. The Git-specific Caddy handler translates Git's error statuses into HTTP statuses and removes the CGI header. A disposable production database/repository copy verified anonymous and invalid credentials return 401, authenticated non-owners return 403, owner discovery advertises `main=37665e69e1aa954f0dec06b1cafa1612f44dc907`, and an actual `jj git push --bookmark main` advances the clone to `b8e734ce6aabd94a7e1aacfc989467662ca27dbe`. Disabled receive-pack returns actual HTTP 403; invalid method/content type return 400/415 without a `Status` header. No production token was added.
76
77## r1807 JSON upgrade, 2026-10-13 UTC
78
79The pinned image in [service.pkl](../service/shale/service.pkl) serves repository lists, issue lists, and issue details as JSON with `Accept: application/json` on their existing URLs. The MCP reads those responses, retaining issue numbers as `id` and raw Markdown in comment text. Native JSON omits comment IDs and issue-list creation times, so the MCP omits those fields. Account settings return 406 for JSON, and label JSON omits numeric IDs; those two reads retain their HTML checks. Writes still use the host broker for durable model authorship, repository permissions, delegated statuses, and grant revocation.
80
81The inherited `shale-preview-8ff39b2c` contained earlier disposable test writes. Rehearsing its previous pinned image followed by r1807 preserved every row of all 10 existing tables, including 559 issues; SQLite integrity passed and its allocation had zero restarts. Preview mirror cloning and polling are disabled. Public browsing, README/file navigation, open/closed issue filters, issue detail, labels, keyboard navigation, and 390-pixel layouts passed browser checks.
82
83An independent instance used only synthetic databases, credentials, certificates, and native Snowglobe OIDC. The candidate Nix dashboard passed linking, every issue/label operation, raw Markdown and durable model attribution, anonymous/other-user/selected-repository denials, exact-Origin enforcement, uncertain-write handling without replay, 80 concurrent JSON reads, 280 concurrent fenced-Markdown reads, backend/dashboard restart persistence, revocation, unlinking, and SQLite integrity. `NPROC=1` remains in place. The router test also verifies JSON headers and bytes pass through unchanged. Restricted verification evidence is `/var/lib/studio/shale-json-upgrade-20261013`.