1 /*****************************************************************************
3 * Project ___| | | | _ \| |
5 * | (__| |_| | _ <| |___
6 * \___|\___/|_| \_\_____|
9 * Download a document and use libtidy to parse the HTML.
10 * Written by Jeff Pohlmeyer
12 * LibTidy => http://tidy.sourceforge.net
14 * gcc -Wall -I/usr/local/include tidycurl.c -lcurl -ltidy -o tidycurl
19 #include <tidy/tidy.h>
20 #include <tidy/buffio.h>
21 #include <curl/curl.h>
23 /* curl write callback, to fill tidy's input buffer... */
24 uint write_cb(char *in, uint size, uint nmemb, TidyBuffer *out)
28 tidyBufAppend( out, in, r );
32 /* Traverse the document tree */
33 void dumpNode(TidyDoc doc, TidyNode tnod, int indent )
36 for ( child = tidyGetChild(tnod); child; child = tidyGetNext(child) )
38 ctmbstr name = tidyNodeGetName( child );
41 /* if it has a name, then it's an HTML tag ... */
43 printf( "%*.*s%s ", indent, indent, "<", name);
44 /* walk the attribute list */
45 for ( attr=tidyAttrFirst(child); attr; attr=tidyAttrNext(attr) ) {
46 printf(tidyAttrName(attr));
47 tidyAttrValue(attr)?printf("=\"%s\" ",
48 tidyAttrValue(attr)):printf(" ");
53 /* if it doesn't have a name, then it's probably text, cdata, etc... */
56 tidyNodeGetText(doc, child, &buf);
57 printf("%*.*s\n", indent, indent, buf.bp?(char *)buf.bp:"");
60 dumpNode( doc, child, indent + 4 ); /* recursive */
65 int main(int argc, char **argv )
68 char curl_errbuf[CURL_ERROR_SIZE];
70 TidyBuffer docbuf = {0};
71 TidyBuffer tidy_errbuf = {0};
74 curl = curl_easy_init();
75 curl_easy_setopt(curl, CURLOPT_URL, argv[1]);
76 curl_easy_setopt(curl, CURLOPT_ERRORBUFFER, curl_errbuf);
77 curl_easy_setopt(curl, CURLOPT_NOPROGRESS, 0L);
78 curl_easy_setopt(curl, CURLOPT_VERBOSE, 1L);
79 curl_easy_setopt(curl, CURLOPT_WRITEFUNCTION, write_cb);
82 tidyOptSetBool(tdoc, TidyForceOutput, yes); /* try harder */
83 tidyOptSetInt(tdoc, TidyWrapLen, 4096);
84 tidySetErrorBuffer( tdoc, &tidy_errbuf );
87 curl_easy_setopt(curl, CURLOPT_WRITEDATA, &docbuf);
88 err=curl_easy_perform(curl);
90 err = tidyParseBuffer(tdoc, &docbuf); /* parse the input */
92 err = tidyCleanAndRepair(tdoc); /* fix any problems */
94 err = tidyRunDiagnostics(tdoc); /* load tidy error buffer */
96 dumpNode( tdoc, tidyGetRoot(tdoc), 0 ); /* walk the tree */
97 fprintf(stderr, "%s\n", tidy_errbuf.bp); /* show errors */
103 fprintf(stderr, "%s\n", curl_errbuf);
106 curl_easy_cleanup(curl);
107 tidyBufFree(&docbuf);
108 tidyBufFree(&tidy_errbuf);
114 printf( "usage: %s <url>\n", argv[0] );