Load Very Heavy Stream With Gson

I'm trying to read a very heavy JSON (over than 6000 objects) and store them on a hash map to insert it into my database later. But the problem is that I face with OOM and that's c

Solution 1:

From my experience, yes you can use google GSON to stream JSON data this is an example how to do it :

APIModelresult=newAPIModel();
        try {
            HttpResponse response;
            HttpClientmyClient=newDefaultHttpClient();
            HttpPostmyConnection=newHttpPost(APIParam.API_001_PRESENT(
                    serial_id, api_key));
            try {
                response = myClient.execute(myConnection);
                ReaderstreamReader=newInputStreamReader(response
                        .getEntity().getContent());
                JsonReaderreader=newJsonReader(streamReader);
                reader.beginObject();
                while (reader.hasNext()) {

                    Stringname= reader.nextName();

                    if (name.equals("result")) {
                        if (reader.nextString() == "NG") {
                            result.setResult(Util.API_001_RESULT_NG);
                            break;
                        }
                    } elseif (name.equals("items")) {
                        result = readItemsArray(reader);
                    } else {
                        reader.skipValue(); // avoid some unhandle events
                    }
                }

                reader.endObject();
                reader.close();
            } catch (Exception e) {
                e.printStackTrace();
                result.setResult(Util.API_001_RESULT_NG);
            }
        } catch (Exception e) {
            e.printStackTrace();
            result.setResult(Util.API_001_RESULT_NG);
        }

readItemsArray function :

// read items arrayprivate APIModel readItemsArray(JsonReader reader) throws IOException {
        APIModel result = new APIModel();
        String item_name, file_name, data;
        result.setResult(Util.API_001_RESULT_OK);

        reader.beginArray();
        while (reader.hasNext()) {
            item_name = "";
            file_name = "";
            data = "";
            reader.beginObject();
            while (reader.hasNext()) {
                String name = reader.nextName();
                if (name.equals("name")) {
                    item_name = reader.nextString();
                } elseif (name.equals("file")) {
                    file_name = reader.nextString();
                } elseif (name.equals("data")) {
                    data = reader.nextString();
                } else {
                    reader.skipValue();
                }
            }
            reader.endObject();
            result.populateModel("null", item_name, file_name, data);
        }
        reader.endArray();
        return result;
    }

API Model Class :

publicclassAPIModel {
    private int result;
    privateString error_title;
    privateString error_message;
    privateArrayList<String> type;
    privateArrayList<String> item_name;
    privateArrayList<String> file_name;
    privateArrayList<String> data;

    publicAPIModel() {
        result = -1;
        error_title = "";
        error_message = "";
        setType(newArrayList<String>());
        setItem_name(newArrayList<String>());
        setFile_name(newArrayList<String>());
        setData(newArrayList<String>());
    }

    publicvoidpopulateModel(Stringtype, String item_name, String file_name, String data) {
        this.type.add(type);
        this.item_name.add(item_name);
        this.file_name.add(file_name);
        this.data.add(data);
    }

    public int getResult() {
        return result;
    }

    publicvoidsetResult(int result) {
        this.result = result;
    }

    publicStringgetError_title() {
        return error_title;
    }

    publicvoidsetError_title(String error_title) {
        this.error_title = error_title;
    }

    publicStringgetError_message() {
        return error_message;
    }

    publicvoidsetError_message(String error_message) {
        this.error_message = error_message;
    }

    publicArrayList<String> getType() {
        returntype;
    }

    publicvoidsetType(ArrayList<String> type) {
        this.type = type;
    }

    publicArrayList<String> getItem_name() {
        return item_name;
    }

    publicvoidsetItem_name(ArrayList<String> item_name) {
        this.item_name = item_name;
    }

    publicArrayList<String> getFile_name() {
        return file_name;
    }

    publicvoidsetFile_name(ArrayList<String> file_name) {
        this.file_name = file_name;
    }

    publicArrayList<String> getData() {
        return data;
    }

    publicvoidsetData(ArrayList<String> data) {
        this.data = data;
    }


}

before I use the streaming API from google GSON I also got OOM error because the JSON data I got is very big data (many images and sounds in Base64 encoding) but with GSON streaming I can overcome that error because it reads the data per token not all at once. And for Jackson JSON library I think it also have streaming API and how to use it almost same with my implementation with google GSON. I hope my answer can help you and if you have another question about my answer feel free to ask in the comment :)