Madeline_Nokto/Silly-Jirafe
1
0 Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

0.0.1.6

Browse Source
This commit is contained in:
Celeste Madeline Samuel Nokto Dur Grossiord committed 2026-10-11 15:47:17 +02:00
1 parent 97649d73eb
commit e77afd4d29
127 files changed
+7320 -1886
Download Patch File Download Diff File Expand all files Collapse all files

No files matched your search

SillyJirafe.org/.classpath
+2 -2
View File
@@ -6,12 +6,12 @@
</attributes>
</classpathentry>
<classpathentry kind="src" path="src"/>
<classpathentry kind="lib" path="/home/kotama/Silly-Jirafe/libs/sqlite-jdbc-3.53.2.1.jar">
<classpathentry kind="lib" path="libs/json-20250107.jar">
<attributes>
<attribute name="module" value="true"/>
</attributes>
</classpathentry>
<classpathentry kind="lib" path="/home/kotama/Silly-Jirafe/libs/json-20250107.jar">
<classpathentry kind="lib" path="libs/sqlite-jdbc-3.53.2.1.jar">
<attributes>
<attribute name="module" value="true"/>
</attributes>
SillyJirafe.org/bin/org/sillyjirafe/db/Crawl_StateDB.class
BIN
View File
Binary file not shown.
SillyJirafe.org/bin/org/sillyjirafe/linker/Main.class
BIN
View File
Binary file not shown.
SillyJirafe.org/bin/org/sillyjirafe/uoapi/Out.class
BIN
View File
Binary file not shown.
SillyJirafe.org/bin/org/sillyjirafe/uoapi/binds/GitHubApacheApi.class
BIN
View File
Binary file not shown.
SillyJirafe.org/bin/org/sillyjirafe/uoapi/binds/GitHubLGPLApi.class
BIN
View File
Binary file not shown.
SillyJirafe.org/bin/org/sillyjirafe/uoapi/binds/GitHubMITApi.class
BIN
View File
Binary file not shown.
SillyJirafe.org/bin/org/sillyjirafe/uoapi/binds/OpenVerseAudiosApi.class
BIN
View File
Binary file not shown.
SillyJirafe.org/bin/org/sillyjirafe/uoapi/binds/OpenVerseImagesApi.class
BIN
View File
Binary file not shown.
SillyJirafe.org/bin/org/sillyjirafe/uoapi/binds/PixabayImagesApi.class
BIN
View File
Binary file not shown.
SillyJirafe.org/libs/json-20250107.jar
BIN
View File
Binary file not shown.
SillyJirafe.org/libs/sqlite-jdbc-3.53.2.1.jar
BIN
View File
Binary file not shown.
SillyJirafe.org/src/org/sillyjirafe/db/Crawl_StateDB.java
+83 -4
View File
@@ -10,7 +10,7 @@ import java.sql.Statement;
public class Crawl_StateDB
{
String URL = new String();
String insertSql = "INSERT OR IGNORE INTO crawlstates (bind_name, last_page, rnd) VALUES (?, ?, ?)";
String insertSql = "INSERT OR IGNORE INTO crawlstates (bind_name, last_page, rnd, error) VALUES (?, ?, ?, ?)";
Connection connection;
public Crawl_StateDB(String BDstr) throws SQLException
@@ -27,7 +27,8 @@ public class Crawl_StateDB
CREATE TABLE IF NOT EXISTS crawlstates (
bind_name TEXT PRIMARY KEY,
last_page INTEGER NOT NULL DEFAULT 1,
rnd TEXT NOT NULL DEFAULT 'a'
rnd TEXT NOT NULL DEFAULT 'a',
error INTEGER NOT NULL DEFAULT 0,
)
""");
}
@@ -72,6 +73,20 @@ public class Crawl_StateDB
}
}
}
public int getErrors(String bindName) throws SQLException {
String sql = "SELECT error FROM crawlstates WHERE bind_name = ?";
try (PreparedStatement ps = connection.prepareStatement(sql)) {
ps.setString(1, bindName);
ResultSet rs = ps.executeQuery();
if (rs.next()) {
return rs.getInt("error");
} else {
return 0;
}
}
}
public void updatePage(String bindName) throws SQLException {
String sql = """
INSERT INTO crawlstates (bind_name, last_page) VALUES (?, 2)
@@ -83,9 +98,73 @@ public class Crawl_StateDB
ps.executeUpdate();
}
}
public void updateletter(String bindName)
public void setRnd(String bindName, String rnd) throws SQLException {
String sql = """
INSERT INTO crawlstates (bind_name, rnd) VALUES (?, 2)
ON CONFLICT(bind_name) DO UPDATE SET rnd = (rnd)
""";
try (PreparedStatement ps = connection.prepareStatement(sql)) {
ps.setString(1, bindName);
ps.setString(2, rnd);
ps.executeUpdate();
}
}
public void newError(String bindName) throws SQLException {
String sql = """
INSERT INTO crawlstates (bind_name, error) VALUES (?, 2)
ON CONFLICT(bind_name) DO UPDATE SET error = error + 1
""";
try (PreparedStatement ps = connection.prepareStatement(sql)) {
ps.setString(1, bindName);
ps.executeUpdate();
}
}
public void resetError(String bindName) throws SQLException {
String sql = """
INSERT INTO crawlstates (bind_name, error) VALUES (?, 2)
ON CONFLICT(bind_name) DO UPDATE SET error = 0
""";
try (PreparedStatement ps = connection.prepareStatement(sql)) {
ps.setString(1, bindName);
ps.executeUpdate();
}
}
public void resetPage(String bindName) throws SQLException {
String sql = """
INSERT INTO crawlstates (bind_name, last_page) VALUES (?, 2)
ON CONFLICT(bind_name) DO UPDATE SET last_page = 1
""";
try (PreparedStatement ps = connection.prepareStatement(sql)) {
ps.setString(1, bindName);
ps.executeUpdate();
}
}
public static String nextLetter(String current) {
char[] lettres = current.toCharArray();
int i = lettres.length - 1;
while (i >= 0) {
if (lettres[i] < 'z') {
lettres[i]++;
return new String(lettres);
}
lettres[i] = 'a';
i--;
}
return "a" + new String(lettres);
}
public void updateletter(String bindName) throws SQLException
{
if (getErrors(bindName) >= 10)
{
setRnd(bindName, nextLetter(getRND(bindName)));
resetError(bindName);
resetPage(bindName);
}
}
public void Close() throws SQLException
SillyJirafe.org/src/org/sillyjirafe/linker/Main.java
+18 -3
View File
@@ -18,15 +18,26 @@ public class Main {
elementDB.LinkToDB();
crawlstateDB.LinkToDB();
Map<String, Boolean> succes = Out.getSucces();
List<String> bindNames = Out.PreOut();
Map<String, Integer> pages = new HashMap<>();
for (String name : bindNames) {
pages.put(name, crawlstateDB.getPage(name));
}
Map<String, String> rnds = new HashMap<>();
for (String nom : bindNames) {
pages.put(nom, crawlstateDB.getPage(nom));
rnds.put(nom, crawlstateDB.getRND(nom));
}
for (String nom : bindNames) {
crawlstateDB.updateletter(nom);
}
List<element> elements = Out.out(pages, ApiKeys.read());
List<element> elements = Out.out(pages, ApiKeys.read(), rnds);
for (int i = 0; i < elements.size(); i++)
{
@@ -35,7 +46,11 @@ public class Main {
}
for (String name : bindNames) {
crawlstateDB.updatePage(name);
if (succes.getOrDefault(name, false)) {
crawlstateDB.updatePage(name);
} else {
crawlstateDB.newError(name);
}
}
elementDB.Close();
SillyJirafe.org/src/org/sillyjirafe/uoapi/Out.java
+14 -7
View File
@@ -9,6 +9,7 @@ import org.sillyjirafe.uoapi.binds.*;
public class Out {
List<Api> Apis = new ArrayList<>();
Map<String, Boolean> Succes = new HashMap<>();
public List<String> PreOut()
{
@@ -29,18 +30,24 @@ public class Out {
return bindnames;
}
public List<element> out(Map<String, Integer> bindpages, Map<String, String> ApiKeys) throws Exception
public List<element> out(Map<String, Integer> bindpages, Map<String, String> ApiKeys, Map<String, String> Rnds) throws Exception
{
List<element> Elements = new ArrayList<>();
for (int i = 0; i < Apis.size(); i++)
{
List<element> currentapi = Caller.CallApi(Apis.get(i), bindpages.get(Apis.get(i).getName()), ApiKeys.get(Apis.get(i).getName()));
for (int j = 0; j < currentapi.size(); j++)
{
Elements.add(currentapi.get(j));
for (Api api : Apis) {
try {
List<element> currentapi = Caller.CallApi(api, bindpages.get(api.getName()), ApiKeys.get(api.getName()), Rnds.get(api.getName()));
Elements.addAll(currentapi);
Succes.put(api.getName(), true);
} catch (Exception e) {
System.out.println("Error " + api.getName() + " : " + e.getMessage());
Succes.put(api.getName(), false);
}
}
return Elements;
}
public Map<String, Boolean> getSucces() {
return Succes;
}
}
SillyJirafe.org/src/org/sillyjirafe/uoapi/binds/GitHubApacheApi.java
+2 -6
View File
@@ -15,14 +15,13 @@ public class GitHubApacheApi implements Api
return "GitHubApacheApi";
}
public String getURL(int page, String apikey) {
return "https://api.github.com/search/repositories?q=license:apache-2.0&page=" + page;
public String getURL(int page, String apikey, String rnd) {
return "https://api.github.com/search/repositories?q="+ rnd +"&license:apache-2.0&page=" + page;
}
public List<element> Parse(String jsonRaw) {
List<element> resultats = new ArrayList<>();
try {
JSONObject json = new JSONObject(jsonRaw);
JSONArray items = json.getJSONArray("items");
@@ -35,9 +34,6 @@ public class GitHubApacheApi implements Api
resultats.add(new element(titre, url, licence, "github", "code"));
}
} catch (org.json.JSONException e) {
System.out.println("error but continue : " + e.getMessage());
}
return resultats;
}
SillyJirafe.org/src/org/sillyjirafe/uoapi/binds/GitHubLGPLApi.java
+2 -6
View File
@@ -16,14 +16,13 @@ public class GitHubLGPLApi implements Api
return "GitHubLGPLApi";
}
public String getURL(int page, String apikey) {
return "https://api.github.com/search/repositories?q=license:lgpl-3.0&page=" + page;
public String getURL(int page, String apikey, String rnd) {
return "https://api.github.com/search/repositories?q="+ rnd + "&license:lgpl-3.0&page=" + page;
}
public List<element> Parse(String jsonRaw) {
List<element> resultats = new ArrayList<>();
try {
JSONObject json = new JSONObject(jsonRaw);
JSONArray items = json.getJSONArray("items");
@@ -36,9 +35,6 @@ public class GitHubLGPLApi implements Api
resultats.add(new element(titre, url, licence, "github", "code"));
}
} catch (org.json.JSONException e) {
System.out.println("error but continue : " + e.getMessage());
}
return resultats;
}
SillyJirafe.org/src/org/sillyjirafe/uoapi/binds/GitHubMITApi.java
+2 -6
View File
@@ -15,15 +15,14 @@ public class GitHubMITApi implements Api
return "GitHubMITApi";
}
public String getURL(int page, String apikey) {
return "https://api.github.com/search/repositories?q=license:mit&page=" + page;
public String getURL(int page, String apikey, String rnd) {
return "https://api.github.com/search/repositories?q="+ rnd +"&license:mit&page=" + page;
}
public List<element> Parse(String jsonRaw) {
List<element> resultats = new ArrayList<>();
try {
JSONObject json = new JSONObject(jsonRaw);
JSONArray items = json.getJSONArray("items");
@@ -36,9 +35,6 @@ public class GitHubMITApi implements Api
resultats.add(new element(titre, url, licence, "github", "code"));
}
} catch (org.json.JSONException e) {
System.out.println("error but continue : " + e.getMessage());
}
return resultats;
}
SillyJirafe.org/src/org/sillyjirafe/uoapi/binds/OpenVerseAudiosApi.java
+2 -6
View File
@@ -15,14 +15,13 @@ public class OpenVerseAudiosApi implements Api
return "OpenVerseAudiosApi";
}
public String getURL(int page, String apikey) {
return "https://api.openverse.org/v1/audio/?format=json&page=" + page;
public String getURL(int page, String apikey, String rnd) {
return "https://api.openverse.org/v1/audio/?q=\""+ rnd +"\"format=json&page=" + page;
}
public List<element> Parse(String jsonRaw) {
List<element> resultats = new ArrayList<>();
try {
JSONObject json = new JSONObject(jsonRaw);
JSONArray items = json.getJSONArray("results");
@@ -35,9 +34,6 @@ public class OpenVerseAudiosApi implements Api
resultats.add(new element(titre, url, licence, "openverse", "audio"));
}
} catch (org.json.JSONException e) {
System.out.println("error but continue : " + e.getMessage());
}
return resultats;
}
}
SillyJirafe.org/src/org/sillyjirafe/uoapi/binds/OpenVerseImagesApi.java
+2 -7
View File
@@ -15,14 +15,12 @@ public class OpenVerseImagesApi implements Api
return "OpenVerseImagesApi";
}
public String getURL(int page, String apikey) {
return "https://api.openverse.org/v1/images/?format=json&page=" + page;
public String getURL(int page, String apikey, String rnd) {
return "https://api.openverse.org/v1/images/?q=\"" + rnd+"\"format=json&page=" + page;
}
public List<element> Parse(String jsonRaw) {
List<element> resultats = new ArrayList<>();
try {
JSONObject json = new JSONObject(jsonRaw);
JSONArray items = json.getJSONArray("results");
@@ -35,9 +33,6 @@ public class OpenVerseImagesApi implements Api
resultats.add(new element(titre, url, licence, "openverse", "image"));
}
} catch (org.json.JSONException e) {
System.out.println("error but continue : " + e.getMessage());
}
return resultats;
}
}
SillyJirafe.org/src/org/sillyjirafe/uoapi/binds/PixabayImagesApi.java
+1 -5
View File
@@ -15,14 +15,13 @@ public class PixabayImagesApi implements Api
return "PixabayImagesApi";
}
public String getURL(int page, String apikey) {
public String getURL(int page, String apikey, String rnd) {
return "https://pixabay.com/api/?per_page=200&page=" + page + "&key=" + apikey;
}
public List<element> Parse(String jsonRaw) {
List<element> resultats = new ArrayList<>();
try {
JSONObject json = new JSONObject(jsonRaw);
JSONArray items = json.getJSONArray("hits");
@@ -35,9 +34,6 @@ public class PixabayImagesApi implements Api
resultats.add(new element(titre, url, licence, "pixabay", "image"));
}
} catch (org.json.JSONException e) {
System.out.println("error but continue : " + e.getMessage());
}
return resultats;
}