8000 Parser-correctness sweep, batch 1: streaming attribution link, empty background, search synopsis by irfan-dahir · Pull Request #676 · jikan-me/jikan · GitHub
[go: up one dir, main page]

Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
5 changes: 5 additions & 0 deletions CHANGELOG.md
Original file line number Diff line number Diff line change
Expand Up @@ -96,6 +96,11 @@ which changes a consumer can see.
### Fixed
- **A person's anime staff positions** (`AnimeStaffPosition::getPosition()`) started with the text of MyAnimeList's "add" (to list) button, e.g. `add Director, Storyboard`; they now hold the position alone, e.g. `Director, Storyboard`.
- **A producer's external links** (`Producer::getExternalLinks()`) were empty when its page had only one of the "Available At" and "Resources" sections; the links of the section present are now returned.
- **Streaming platform links** (`Anime::getStreamingLinks()`) included MyAnimeList's "Streaming platforms provided by JustWatch" attribution link: an extra entry with no platform name and a `justwatch.com` URL on every page that lists platforms. Only platforms are returned now.
- **`Anime::getBackground()`** returned `''` for a title whose sidebar has the background section but no text in it; it returns `null`, as it already did for a title with no background section at all.

### Changed
- **`AnimeSearchListItem::getSynopsis()` and `MangaSearchListItem::getSynopsis()`** return `null` when MyAnimeList shows a search result with no synopsis (the page carries an empty synopsis block), instead of `''`. These were the last search fields that reported absence as an empty string rather than `null`.

---

Expand Down
6 changes: 3 additions & 3 deletions src/Model/Search/AnimeSearchListItem.php
Original file line number Diff line number Diff line change
Expand Up @@ -38,7 +38,7 @@ class AnimeSearchListItem
private $airing;

/**
* @var string
* @var string|null
*/
private $synopsis;

Expand Down Expand Up @@ -139,9 +139,9 @@ public function getTitle(): string
}

/**
* @return string
* @return string|null
*/
public function getSynopsis(): string
public function getSynopsis(): ?string
{
return $this->synopsis;
}
Expand Down
6 changes: 3 additions & 3 deletions src/Model/Search/MangaSearchListItem.php
Original file line number Diff line number Diff line change
Expand Up @@ -39,7 +39,7 @@ class MangaSearchListItem
private $publishing;

/**
* @var string
* @var string|null
*/
private $synopsis;

Expand Down Expand Up @@ -151,9 +151,9 @@ public function getTitle(): string
}

/**
* @return string
* @return string|null
*/
public function getSynopsis(): string
public function getSynopsis(): ?string
{
return $this->synopsis;
}
Expand Down
27 changes: 25 additions & 2 deletions src/Parser/Anime/AnimeParser.php
Original file line number Diff line number Diff line change
Expand Up @@ -9,6 +9,7 @@
use Jikan\Model\Anime\Anime;
use Jikan\Model\Common\DateRange;
use Jikan\Model\Common\MalUrl;
use Jikan\Model\Common\Url;
use Jikan\Model\Common\Title;
use Jikan\Parser\Common\AlternativeTitleParser;
use Jikan\Parser\Common\MalUrlParser;
Expand Down Expand Up @@ -723,10 +724,28 @@ public function getStreamingLinks(): array
return [];
}

return $links->filterXPath('//a')
return $this->streamingLinks($links);
}

/**
* @param Crawler $links
* @return \Jikan\Model\Common\Url[]
*/
private function streamingLinks(Crawler $links): array
{
$models = $links->filterXPath('//a')
->each(function (Crawler $c) {
return new UrlParser($c)->getModel();
});

// The block ends with a JustWatch attribution link ("Streaming platforms provided by
// JustWatch"), which is not a platform and carries no name.
return array_values(
array_filter(
$models,
static fn(Url $url): bool => $url->getName() !== ''
)
);
}

/**
Expand Down Expand Up @@ -807,7 +826,11 @@ public function getBackground(): ?string
return null;
}

return JString::cleanse($background);
// The section can be present and still hold no text (the paragraph is removed above), which
// is an absent background, not an empty one — the model's getter is ?string for exactly this.
$background = JString::cleanse($background);

return $background === '' ? null : $background;
}

/**
Expand Down
10 changes: 7 additions & 3 deletions src/Parser/Search/AnimeSearchListItemParser.php
Original file line number Diff line number Diff line change
Expand Up @@ -74,16 +74,20 @@ public function getImageUrl(): ?string
}

/**
* @return string
* @return string|null
* @throws \InvalidArgumentException
*/
public function getSynopsis(): string
public function getSynopsis(): ?string
{
return JString::cleanse(
$synopsis = JString::cleanse(
Parser::removeChildNodes(
$this->crawler->filterXPath('//td[2]/div[@class="pt4"]')
)->text()
);

// A title with no synopsis still gets the div, empty: that is absence, not an empty string
// (DECISIONS.md, "Missing and unreadable data").
return $synopsis === '' ? null : $synopsis;
}

/**
Expand Down
10 changes: 7 additions & 3 deletions src/Parser/Search/MangaSearchListItemParser.php
Original file line number Diff line number Diff line change
Expand Up @@ -73,16 +73,20 @@ public function getImageUrl(): ?string
}

/**
* @return string
* @return string|null
* @throws \InvalidArgumentException
*/
public function getSynopsis(): string
public function getSynopsis(): ?string
{
return JString::cleanse(
$synopsis = JString::cleanse(
Parser::removeChildNodes(
$this->crawler->filterXPath('//td[2]/div[@class="pt4"]')
)->text()
);

// A title with no synopsis still gets the div, empty: that is absence, not an empty string
// (DECISIONS.md, "Missing and unreadable data").
return $synopsis === '' ? null : $synopsis;
}

/**
Expand Down
37 changes: 37 additions & 0 deletions test/unit/Corpus/SearchSynopsisTest.php
Original file line number Diff line number Diff line change
@@ -0,0 +1,37 @@
<?php

namespace JikanTest\Corpus;

use Jikan\Model\Search\AnimeSearchListItem;
use Jikan\Parser\Search\AnimeSearchParser;
use JikanTest\TestCase;
use PHPUnit\Framework\Attributes\Test;

/**
* A search result whose title has no synopsis still carries `<div class="pt4"></div>` on the page —
* an empty block, which used to parse as `''`. MyAnimeList not showing a value means `null` in this
* library (DECISIONS.md, "Missing and unreadable data"), so the getter is `?string`.
*/
class SearchSynopsisTest extends TestCase
{
#[Test]
public function a_result_with_an_empty_synopsis_block_is_null(): void
{
$search = (new AnimeSearchParser(Variants::crawler('anime-search-fate')))->getModel();

$withoutSynopsis = array_values(
array_filter($search->getResults(), static fn (AnimeSearchListItem $item): bool => $item->getSynopsis() === null)
);

self::assertCount(1, $withoutSynopsis);
self::assertSame('Unmei', $withoutSynopsis[0]->getTitle());

foreach ($search->getResults() as $item) {
if ($item->getSynopsis() === null) {
continue;
}

self::assertNotSame('', $item->getSynopsis(), $item->getTitle() . ' kept an empty synopsis.');
}
}
}
70 changes: 70 additions & 0 deletions test/unit/Corpus/StreamingLinksTest.php
Original file line number Diff line number Diff line change
@@ -0,0 +1,70 @@
<?php

namespace JikanTest\Corpus;

use Jikan\Model\Common\Url;
use Jikan\Parser\Anime\AnimeParser;
use JikanTest\TestCase;
use PHPUnit\Framework\Attributes\Test;

/**
* Two fields the model invariants flagged as empty in every recording:
*
* - the streaming-platforms block ends with a JustWatch attribution link ("Streaming platforms
* provided by JustWatch"), which is not a platform and carries no name — it was parsed as an
* extra streaming link with `''` for its name;
* - a detail page whose sidebar has no background paragraph produced `''` instead of `null`.
*/
class StreamingLinksTest extends TestCase
{
#[Test]
public function it_does_not_parse_the_justwatch_attribution_link_as_a_platform(): void
{
$links = new AnimeParser(Variants::crawler('anime-1'))->getStreamingLinks();

self::assertNotSame([], $links);
self::assertContainsOnlyInstancesOf(Url::class, $links);

foreach ($links as $link) {
self::assertNotSame('', $link->getName(), 'A streaming link has no platform name.');
self::assertStringNotContainsString(
'//www.justwatch.com/',
$link->getUrl(),
'The JustWatch attribution link is not a streaming platform.'
);
}
}

#[Test]
public function it_returns_null_when_the_sidebar_has_no_background(): void
{
$anime = (new AnimeParser(Variants::crawler('anime-5114')))->getModel();

self::assertNull($anime->getBackground());
}

#[Test]
public function it_keeps_a_background_that_exists(): void
{
// The null path above must not swallow a real background: anime/1 has one.
$anime = (new AnimeParser(Variants::crawler('anime-1')))->getModel();

self::assertIsString($anime->getBackground());
self::assertNotSame('', $anime->getBackground());
}

#[Test]
public function it_filters_the_attribution_on_the_other_streaming_markup(): void
{
// anime/5114 has no #broadcast-block — its platforms hang off div.broadcast anchors instead, so
// this pins the filter against the second shape MyAnimeList serves.
$links = (new AnimeParser(Variants::crawler('anime-5114')))->getStreamingLinks();

self::assertNotSame([], $links);

foreach ($links as $link) {
self::assertNotSame('', $link->getName(), 'A streaming link has no platform name.');
self::assertStringNotContainsString('//www.justwatch.com/', $link->getUrl());
}
}
}
3 changes: 0 additions & 3 deletions test/unit/__invariants__/Corpus/CorpusTest.anime-1.json

This file was deleted.

4 changes: 0 additions & 4 deletions test/unit/__invariants__/Corpus/CorpusTest.anime-11617.json

This file was deleted.

3 changes: 0 additions & 3 deletions test/unit/__invariants__/Corpus/CorpusTest.anime-199.json

This file was deleted.

3 changes: 0 additions & 3 deletions test/unit/__invariants__/Corpus/CorpusTest.anime-21.json

This file was deleted.

4 changes: 0 additions & 4 deletions test/unit/__invariants__/Corpus/CorpusTest.anime-5114.json

This file was deleted.

This file was deleted.

This file was deleted.

This file was deleted.

This file was deleted.

This file was deleted.

This file was deleted.

This file was deleted.

This file was deleted.

This file was deleted.

3 changes: 0 additions & 3 deletions test/unit/__invariants__/Parser/Search/AnimeSearchTest.json

This file was deleted.

This file was deleted.

3 changes: 0 additions & 3 deletions test/unit/__invariants__/Parser/Search/MangaSearchTest.json

This file was deleted.

Loading
0